Merge remote-tracking branch 'origin/master'

This commit is contained in:
2025-07-22 17:47:40 +08:00
27 changed files with 1608 additions and 79 deletions
@@ -29,7 +29,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
@Override
void process(Object args) {
List<String> cities = [];
def allCity = CommonUtils.findAllCity()
def allCity = ["广州","深圳","临沂"]//CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
cities.addAll(allCity)
@@ -38,7 +38,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList()
println(city)
}
ExecutorService executorService = Executors.newFixedThreadPool(20)
ExecutorService executorService = Executors.newFixedThreadPool(1)
districts.each {BasicDBObject district->
executorService.submit(new Calculator(district))
}
@@ -65,22 +65,26 @@ class Calculator extends Thread{
}
def statMap = [:]
DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo->
DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type).append("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01","yyyy-MM-dd")))).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo->
Date weekend = dbo.weekend
if(DateUtil.isLastSundayOfMonth(weekend)){
def trend = new BasicDBObject("city", district.city)
trend.append("region", district.region)
trend.append("district_name", district.name)
trend.append("estate_type", estate_type)
// trend.append("estate_type", estate_type)
trend.append("date", weekend.format("yyyyMM").toInteger())
def udata = new BasicDBObject();
udata.put("avg_price", dbo.avg_price)
udata.put("credit", dbo.credit)
udata.put("m_date", new Date())
DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
statMap.put(weekend.format("yyyyMM").toInteger(), udata)
if(dbo.avg_price_2b){
// udata.put("avg_price", dbo.avg_price)
udata.put("avg_price", dbo.avg_price_2b)
udata.put("credit", dbo.credit)
udata.put("m_date", new Date())
DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
statMap.put(weekend.format("yyyyMM").toInteger(), udata)
}
}
}
DISTRICT_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).sort(new BasicDBObject("date", 1)).each {BasicDBObject trend->
@@ -88,14 +92,14 @@ class Calculator extends Thread{
def monDate = getMomDate(date)
def yoyDate = getYoyDate(date)
BasicDBObject monDbo = statMap.get(monDate)
if(monDbo != null && monDbo.avg_price != 0) {
trend.mom = DataUtil.roundDouble(((trend.avg_price - monDbo.avg_price)/(double)monDbo.avg_price),4)
if(monDbo != null && monDbo.avg_price_2b != 0) {
trend.mom = DataUtil.roundDouble(((trend.avg_price_2b - monDbo.avg_price_2b)/(double)monDbo.avg_price_2b),4)
}else{
trend.mom = 0d
}
BasicDBObject yoyDbo = statMap.get(yoyDate)
if(yoyDbo != null && yoyDbo.avg_price != 0){
trend.yoy = DataUtil.roundDouble(((trend.avg_price - yoyDbo.avg_price)/(double)yoyDbo.avg_price),4)
trend.yoy = DataUtil.roundDouble(((trend.avg_price_2b - yoyDbo.avg_price_2b)/(double)yoyDbo.avg_price_2b),4)
}else{
trend.yoy = 0d
}
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年5月深圳成交明细.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -855,6 +855,32 @@ class CommonDatautils {
}
public static def findAllDistrictsKey(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","经纬天地园二期")
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def keyMap = [:]
for(def d : dbsour){
keyMap.put(d.region+"##"+d.name, d)
}
return keyMap;
}
// public static def findAllDistrictByCity(def city, def cat, def fields){
//
// DBObject q = new BasicDBObject();
@@ -73,6 +73,7 @@ class DistrictPriceExportBatch extends BaseBatch{
def appendResult(def headers, def ret){
def sb = new StringBuffer();
// sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator"));
@@ -121,11 +122,16 @@ class DistrictPriceExportBatch extends BaseBatch{
def day = new Date().getDate()
def startMDate = DateUtils.truncateDay(new Date()-1)
def endMDate = DateUtils.truncateDay(new Date())
if(day == 15){
if(CommonDatautils.isWindow()){
districts = getFzDistricts(city, null, null)
}else{
districts = getFzDistricts(city, startMDate, endMDate)
} else{
if(day == 15){
districts = getFzDistricts(city, null, null)
}else{
districts = getFzDistricts(city, startMDate, endMDate)
}
}
def curPriceMap = findFzDistrictSellStat("福州",curDate)
def result = []
for(def rMap : districts){
@@ -1,5 +1,6 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
@@ -63,7 +64,7 @@ class DistrictSellStatBatch extends BaseBatch {
}
def calcMonthlyPrice(){
//只有福州使用tob价格
//只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外
def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
@@ -29,11 +29,11 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
// task.priceMarkupRun();
// task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
// task.testNewPrice()
task.testNewPrice()
}
def margeExcel(){
@@ -82,7 +82,7 @@ class FzDataResultStat {
}
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -4, "yyyyMM")))
def map2 = [:]
for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name
@@ -119,7 +119,7 @@ class FzDataResultStat {
index++
println row
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
ExcelUtils.write("D:\\数据\\福州\\价格比对5_${curMonth}.xlsx", rows, "数据")
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
@@ -301,16 +301,13 @@ class FzDataResultStat {
}
def testNewPrice(){
ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d->
ExcelUtils.modify("D:\\数据\\福州\\福州200个小区抽样价格核验202507.xlsx",{d->
if(d['A']){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region",d['B'])
q.put("name",d['C'])
q.put("month", 202402)
q.put("comm_id", d['A'])
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){
d['S'] = obj.avg_price
d['D'] = obj.avg_price
}
}
})
@@ -13,11 +13,17 @@ import com.mongodb.DBCursor
import com.mongodb.DBObject
import org.apache.solr.common.util.DateUtil
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AddAvgpriceBatch {
private static def citys = null;
private static def districtMap = [:];
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
static {
citys = ["宜昌"]//CommonDatautils.findAllCity()
@@ -26,10 +32,112 @@ class AddAvgpriceBatch {
static main(args) {
def task = new AddAvgpriceBatch();
task.run1();
// task.run1();
// task.test()
// task.sendPriceCommand()
// task.sendNewDistPrice()
// task.delNoExistsDistPrice()
task.checkToBprice()
}
def checkToBprice(){
def citys = CommonDatautils.findAllCity().reverse()
for(def i=0;i<citys.size();i++){ //76
def c = citys[i]
def keyDistMap = CommonDatautils.findAllDistrictsKey(c, "district", ["city","region","name"])
def rets = findNoPrices(c, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), keyDistMap)
println c + " "+rets.size()
addCommand(rets)
}
}
def addCommand(rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
def m = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
CommonUtils.recalculationAverage(m.city, m.district, m.name, m.estate_type, m.weekend,false,0)
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findNoPrices(c, weekend, keyDistMap){
def q = new BasicDBObject("city", c).append("weekend", new BasicDBObject('$gte', weekend)).append("avg_price_2b", null)
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("district_name", 1)
keys.put("estate_type", 1)
keys.put("avg_price_2b", 1)
keys.put("weekend", 1)
def dbcursor = DIST_STAT_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbcursor){
def distObj = keyDistMap.get(d.region+"##"+d.district_name)
if(distObj){
if(!d.avg_price_2b) {
rets << ["city": d.city, "district": d.region, "name": d.district_name,"estate_type":d.estate_type, "weekend": d.weekend]
}
}
}
return rets
}
def delNoExistsDistPrice(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
index ++
if(index<250){
continue
}
println index +" "+c
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name"])
def nameKeys = []
for(def dist : dists){
nameKeys << dist.region+"#"+dist.name
}
delDistPrice(c, nameKeys)
}
}
def delDistPrice(city, nameKeys){
def q = new BasicDBObject("city", city)
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("district_name", 1)
def dbcursor = DIST_STAT_COLL.find(q, keys).batchSize(5000)
for(def d : dbcursor){
def key = d.region +"#"+d.district_name
if(!nameKeys.contains(key)){
DIST_STAT_COLL.remove(new BasicDBObject("_id", d._id))
println "删除:"+city +" "+ d.district_name
}
}
}
def sendNewDistPrice(){
@@ -1,21 +1,225 @@
package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.poi.xwpf.usermodel.XWPFDocument
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.Semaphore
import java.util.concurrent.atomic.AtomicLong
class DistrictStat {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new DistrictStat();
// task.run()
task.run("无锡")
// task.run("无锡")
// 合肥市
// 芜湖市
// 安庆市
// 马鞍山市
// 宣称市
// 六安市
// def citys = ["宣城","合肥","芜湖","安庆","马鞍山","六安"]
// for(def city : citys){
// task.stat(city)
// }
// task.cityStat()
task.cityStat1()
}
def cityStat1(){
def index = 0
def rets = []
ExcelUtils.modify("D:\\数据\\小区\\城市数量统计1.xlsx",{d->
def districts = null
if(d['D']=="义乌"){
districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region =="义乌"){
newDists << dist
}
}
districts = newDists
}else if(d['D']=="金华"){
districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region !="义乌"){
newDists << dist
}
}
districts = newDists
}else if(d['D']=="无锡"){
districts = CommonDatautils.findAllDistrictByCity("无锡", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region !="江阴"){
newDists << dist
}
}
districts = newDists
}else if(d['D']=="苏州"){
districts = CommonDatautils.findAllDistrictByCity("苏州", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region !="常熟"){
newDists << dist
}
}
districts = newDists
}else if(d['D']=="济南"){
districts = CommonDatautils.findAllDistrictByCity("济南", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region !="莱芜"){
newDists << dist
}
}
districts = newDists
}else{
districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["region","name"])
}
d['F'] = districts.size()
if(districts){
d['E'] = "是"
}else{
d['E'] = "否"
}
def prov = null
if(districts){
index ++
def data = fullCityMap.get(d['D'])
if(data){
prov = data.province
}
def str = districts.collect { it.name }.join(", ")
rets << ["id":index, "type":"A", "province":prov, "city":d['D'], "names":str]
createDoc(rets)
}
}, 0)
// FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表1.dat", [], "utf-8", false);
}
def createDoc(dataList){
// 创建 Word 文档
def doc = new XWPFDocument()
def table = doc.createTable(dataList.size() + 1, 5) // 行数=数据行+表头,列数=3
// 设置表头
def headerRow = table.getRow(0)
headerRow.getCell(0).setText("序号")
headerRow.getCell(1).setText("类型")
headerRow.getCell(2).setText("省份")
headerRow.getCell(3).setText("城市")
headerRow.getCell(4).setText("楼盘名称")
// 填充数据
dataList.eachWithIndex { item, index ->
def row = table.getRow(index+1) // 跳过表头
row.getCell(0).setText(item.id.toString())
row.getCell(1).setText(item.type.toString())
row.getCell(2).setText(item.province.toString())
row.getCell(3).setText(item.city.toString())
row.getCell(4).setText(item.names.toString())
}
// 保存到文件
def outputFile = new File("A类城市楼盘列表.docx")
doc.write(new FileOutputStream(outputFile))
println "Word 表格生成成功: ${outputFile.absolutePath}"
}
def cityStat(){
def index = 0
def rets = []
ExcelUtils.modify("D:\\数据\\小区\\城市数量统计.xlsx",{d->
def districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["city", "region", "name"])
if(districts){
d['F'] = districts.size()
d['E'] = "是"
rets << ["序号","类型","省份","城市","楼盘名称","金华"]
def data = fullCityMap.get(d['D'])
def prov = null
if(data){
prov = data.province
}
for(def dist : districts){
index++
rets << [index, "A", prov,dist.city, dist.name, dist.region]
}
}
}, 0)
// ExcelUtils.write("D:\\数据\\小区\\A类城市楼盘列表.xlsx", rets, "楼盘数据详情列表")
FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表.dat", [], "utf-8", false);
}
def stat(city){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name","alias"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def result = Collections.synchronizedList(new ArrayList());
def countResult = Collections.synchronizedList(new ArrayList());
def rets = []
// def houseTotal = CommonDatautils.findHouseCountByDate(city, null, null, null, "sell", 20240611)
rets << city
rets << districts.size()
// rets << houseTotal
for(def i=0;i<districts.size();i++){
final def o = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20240611)
if(count){
result << o
countResult << count
}
println totalCount.get() +" "+o.region +" "+o.name+" "+count
} catch (Exception e) {
println totalCount.get() +" 查房源Error: "+city+" "+ o.region+" "+o.name+" "+e.getMessage()
} finally {
latch.countDown()
}
};
})
}
latch.await()
rets << countResult.sum()
rets << result.size()
println rets
}
def run(def city){
def weekend = DateUtils.parseFromTightYYYYMMDD("2020-10-25")
@@ -64,7 +64,9 @@ class FangjiatongUserStat {
def stat(){
def users = findUses(12)
def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
// def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
// def months = ["2020-01-01","2020-02-01","2020-03-01","2020-04-01","2020-05-01","2020-06-01","2020-07-01","2020-08-01","2020-09-01","2020-10-01","2020-11-01","2020-12-01","2021-01-01"]
def months = ["2022-01-01","2022-02-01","2022-03-01","2022-04-01","2022-05-01","2022-06-01","2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01"]
def rets = []
def header = ["用户id","支行名称", "备注名称"]
months.each{m->
@@ -87,7 +89,7 @@ class FangjiatongUserStat {
}
rets << ret
}
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019")
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2022")
}
def findUses(role){
@@ -27,10 +27,29 @@ class SuzouPocTest {
static main(args) {
def task = new SuzouPocTest()
// task.run()
task.newRun()
// task.newRun()
task.test()
}
def test(){
ExcelUtils.modify("D:\\数据\\上海联合产权交易所\\第1批拟处置房产清单(长城大厦、台山碧桂园).xlsx", { d ->
if(d['O']){
def floorAdds = d['I'].split("/")
def ret1 = OpenApiClient.addressEvaluation(d['O'], null, d['D'], d['D'], d['K'], floorAdds[0], floorAdds[1])
if (ret1) {
d['N'] = ret1.totalPrice
d['M'] = ret1.avgPrice
d['P'] = ret1.district
d['Q'] = ret1.name
}
}
});
}
def newRun(){
println("start:"+new Date().toString())
def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls"
newMatch(filePath)
@@ -21,7 +21,7 @@ class FileUtils {
def sb = new StringBuffer();
if(headers){
sb.append(headers.join("|@|"))
sb.append(headers.join("@"))
sb.append("\n");
}
@@ -32,7 +32,7 @@ class FileUtils {
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), "|@|"))
sb.append(StringUtils.join(row.toArray(), "@"))
sb.append("\n");
if(index%10000==0){
@@ -0,0 +1,583 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.newDistrict.InvokeHttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
import net.sf.json.groovy.JsonSlurper
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
class SzGovHouseCrawl {
static main(args) {
def task = new SzGovHouseCrawl();
task.run();
}
def run(){
def index_content = """
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>一手房源公示</title>
<link href="css/marster.css" rel="stylesheet" type="text/css" />
</head>
<body>
<form name="Form1" method="post" action="index.aspx" id="Form1">
<div>
<input type="hidden" name="__EVENTTARGET" id="__EVENTTARGET" value="" />
<input type="hidden" name="__EVENTARGUMENT" id="__EVENTARGUMENT" value="" />
<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="nFQUad+408Aq5zCXFL9mwJbwfbUjHqOTwTmTIzIV/nn2wuuQIXExg/fWXnK8Ll6o8tHsGFJtGbKc87jIO8eGWmN56r/iP2F/Oi3sPuUM14rN/OFTR/y1yS4eQXiTUniZCpvc+hwWUXM6La+cm63hbAskQFo2Geb/8JeX/kCREa/vJqSoFTxJFuxoDevNRV5QeytYQ8PxV89z/L/KAGHwo7sR66wzuWciXwvLcXa6BWaA9aU953Rr1QnFrqy3QrAnmLmxfEtEbvw6cySgECeARwElfHdG0dKGni1inFrFf3lYDdNAL2gLoQoNFBIbDj/vXaYZQoAypynsqc8hiZotSa5aqehFlDxDu3TO3jRcWKQDue55UoGZURl8WyjlfLpebiAdSQirRRrLRQHcb4Baf7U0y/JiFFk0A75I9bfJM72eq16jjjhV9++DNXJRUn5KGMSn5h+PxwcsS8rSga0KRwsM9LDylCsNkedVucDhMCGrNwmbhAsnUIiMD7Va8wQizQllRtDJWPFZRU+YPUWxqRkfJnTEzdl8MQw7eEC6rweywfrDn3VLvw8RkRqAdaSyA5K3z4VXR0QVuBcX6teex3UIQQep9oWVvK6FGlpHEfkqEPYdwnyoEoGgRJHk/R0xhCx2ahrlYI04dbT2GaugM2JDVBnGWhQBYeylLukCVpBnaxUgiGG1Dx6lUBbLi0yFt3GkdYzi5XeTzQoGsMTbdVMBvcIqqFOs9dZBfebrWrpwTnAgUldkrtyw6j9Lf0sKPCUifvCCg8Ks1OnniSkpI1zJ0sfseGrnPQv7357IEg6UWyfvMFr3jjzqcX0QV35ajJMpzfaKIYGlzzIkQwEfjhNU83+nhI3yct8nyfmE2lROKDzOdCYFmlOPVQ1yF0vdPWs2cmx3SeQl53r1nPokHRQFuYKif4B7Tx/GN/1jPoKtrqBekL/8+ewdF1YAL8z7okg4LQ7DWaITUPvZmP7p/kAqilgKVMzqAOmdl5FzK1RwDn1fh6Ciw79aM3MPYwzkfxwA9S9uo67RuF4DHbVkuob/SJ1JGfXvFNJEvm/45UJWBFKEYWnmJJLvbeZSSZ455ox0aNLtzsTpTm3yr7S+LVNqTQIxEdFMed1gC/9caV753lkuEAHlxWwZHE3eULOw04aM8w==" />
</div>
<script type="text/javascript">
//<![CDATA[
var theForm = document.forms['Form1'];
if (!theForm) {
theForm = document.Form1;
}
function __doPostBack(eventTarget, eventArgument) {
if (!theForm.onsubmit || (theForm.onsubmit() != false)) {
theForm.__EVENTTARGET.value = eventTarget;
theForm.__EVENTARGUMENT.value = eventArgument;
theForm.submit();
}
}
//]]>
</script>
<script src="/ris/WebResource.axd?d=oAb-PGCUASjSAEHmsWLcW3nRSbp0kTk2wHZPA_xOoR1TX097F_c8lWeB0anDU_zTJ7nGX_3dSaDy8zzgj1ZXo1L1Sc41&amp;t=638314433260000000" type="text/javascript"></script>
<script src="/ris/ScriptResource.axd?d=ez7z8Rk-sRUjmvQcOq01fhxlJ1QkOJwcsZyID_Ni3ynenZjRnecb_-APpFxKSgOMhcW8F7rusL02bjz8mZvC6mdZ5fFUnv7hA8P09LlmyTRz8AzNNpmFk65QF_tFzWpkOb3lJg2&amp;t=636978664134464033" type="text/javascript"></script>
<script src="/ris/ScriptResource.axd?d=3ACyCjbvPx1mr6XvFjwUpUrueEjvCnwM6jay2gUKFWC2bAv9PJDw7_w8e-FijIouXVInB0UX5vw0I3Fbx5YA92gSlMgpLLv3eu8tE_KjYAT1gVhsDve7JfQShCjQ8wBnrY5v-Q2&amp;t=636978664134464033" type="text/javascript"></script>
<div>
\t<input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="2A35A6B2" />
\t<input type="hidden" name="__VIEWSTATEENCRYPTED" id="__VIEWSTATEENCRYPTED" value="" />
\t<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="c/yjhDZvdy6HmhcL4EkG1J1Z3q6SMKFIycFC2eMEcd9PHzbGDxG56wrvw1REP793VkDqjnfrk1++ArpjUL0ql/yAB3MB5SUwn7h0YzAXw3/YG2ipVYdaOarImjt0o8JYuszLTaNRVxXmK8KTgUXOhpL0mroOqigxEaXUaeSKml9LBl2xzJttjM+uErnLyCLXSyLW0qjXWs+pCPqD5SgIhBUuVmg=" />
</div>
<div class="wrap">
<div class="record fix mb10">
<script type="text/javascript">
//<![CDATA[
Sys.WebForms.PageRequestManager._initialize('scriptManager2', document.getElementById('Form1'));
Sys.WebForms.PageRequestManager.getInstance()._updateControls(['tupdatepanel2'], [], [], 90);
//]]>
</script>
<div class="recordLinkbox recordLink">
所在区:<a id="hypAll" href="javascript:__doPostBack('hypAll','')" style="color:Red;">全市</a>
<a id="hypBa" href="javascript:__doPostBack('hypBa','')" style="color:Black;">宝安</a>
<a id="hypFt" href="javascript:__doPostBack('hypFt','')" style="color:Black;">福田</a>
<a id="hypLg" href="javascript:__doPostBack('hypLg','')" style="color:Black;">龙岗</a>
<a id="hypLh" href="javascript:__doPostBack('hypLh','')" style="color:Black;">罗湖</a>
<a id="hypNs" href="javascript:__doPostBack('hypNs','')" style="color:Black;">南山</a>
<a id="hypQh" href="javascript:__doPostBack('hypQh','')" style="color:Black;">前海</a>
<a id="hypYt" href="javascript:__doPostBack('hypYt','')" style="color:Black;">盐田</a>
<a id="hypLhQ" href="javascript:__doPostBack('hypLhQ','')" style="color:Black;">龙华</a>
<a id="hypGm" href="javascript:__doPostBack('hypGm','')" style="color:Black;">光明</a>
<a id="hypPs" href="javascript:__doPostBack('hypPs','')" style="color:Black;">坪山</a>
<a id="hypDp" href="javascript:__doPostBack('hypDp','')" style="color:Black;">大鹏</a>
<a id="hypSSHZ" href="javascript:__doPostBack('hypSSHZ','')">深汕合作</a></div>
<div class="deveBox" style="line-height: 0px;">
<p class="left mr25">
项目查询 <input name="tep_name" type="text" id="tep_name" style="height:28px;width:150px;" /></p>
<p class="left mr25">
开发企业 <input name="organ_name" type="text" value="深圳市城建产业园发展有限公司" id="organ_name" style="height:28px;width:150px;" /></p>
<p class="left mr25">
宗地地址 <input name="site_address" type="text" id="site_address" style="height:28px;width:150px;" /></p>
</div>
<input type="submit" name="Button1" value="查询" id="Button1" style="background: #1956c6;
width: 75px; height: 28px; line-height: 28px; display: block; text-align: center; cursor:pointer;
font-size: 14px; color: #fff;border-radius: 2px;" />
</div>
<div id="updatepanel2">
\t
<div class="record fix">
<div class="fix">
<table class="table ta-c bor-b-1 table-white">
<tr>
<th style="width: 92px;">
序号
</th>
<th style="width: 220px;">
预售证号
</th>
<th style="width: 312px;">
项目名称
</th>
<th style="width: 270px;">
开发企业
</th>
<th style="width: 110px;">
所在区
</th>
<th style="width: 132px;">
批准时间
</th>
</tr>
<tr class="">
<td>
1
</td>
<td>
<a href='./certdetail.aspx?id=133423' target='_self'>
深房许字(2024)罗湖002号</a>
</td>
<td>
<a href='projectdetail.aspx?id=133423' target='_self'>
凌云华府</a>
</td>
<td>
深圳市城建产业园发展有限公司
</td>
<td>
罗湖
</td>
<td>
2024-06-27
</td>
</tr>
<tr class="bg-grayf7">
<td>
2
</td>
<td>
<a href='./certdetail.aspx?id=124226' target='_self'>
深房许字(2022)罗湖007号</a>
</td>
<td>
<a href='projectdetail.aspx?id=124226' target='_self'>
高投大厦</a>
</td>
<td>
深圳市城建产业园发展有限公司
</td>
<td>
罗湖
</td>
<td>
2022-12-16
</td>
</tr>
<tr style="height:18px;">
<td colspan="6">
</td>
</tr>
</tr></table>
</div>
<div class="page">
<div class="pages pagehtml right">
\t<!-- AspNetPager V7.0.2 for VS2005 & VS2008 Copyright:2003-2007 Webdiyer (www.webdiyer.com) -->
\t<!--记录总数只有一页,AspNetPager已自动隐藏,若需在只有一页数据时显示AspNetPager,请将AlwaysShow属性值设为true!-->
\t<!-- AspNetPager V7.0.2 for VS2005 & VS2008 End -->
\t
</div>
</div>
</div>
</div>
</div>
</div>
<script type="text/javascript">
//<![CDATA[
Sys.Application.initialize();
//]]>
</script>
</form>
</body>
</html>
"""
getRoomRefDetail("https://zjj.sz.gov.cn/ris/bol/szfdc/housedetail.aspx?id=1979350",[:])
def doc = Jsoup.parse(index_content);
Elements trs = doc.select("table tr")
def name = "深圳市城建产业园发展有限公司"
def results = []
def header = ["楼层","房间号","单元号","销售状态","楼栋名称","预售许可证号","项目名称","所属项目地址","开发商名称","开盘日期","监管银行","监管账户名称","监管账户账号","房屋用途","户型","预测总建筑面积","预测套内建筑面积","预测公摊建筑面积","实际总建筑面积","实际套内建筑面积","实际公摊建筑面积","城市","拟售价格-按建筑面积计","拟售价格-按套内建筑面积计","预售总套数","现售总套数","栋数"]
results << header
//开发商查询结果页
println trs.size()-2 +" 预售证"
for(def i=1 ;i<trs.size();i++){
def href1 = trs[i].select("td:eq(1) a").attr("href")
if(href1){
href1 = "https://zjj.sz.gov.cn/ris/bol/szfdc"+ href1.substring(1)
def href2 = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+trs[i].select("td:eq(2) a").attr("href")
def sellObjMap = getSellCodeObject(href1) //预售证页面
println i+ " 预售证页面 加载完成"
def projectMap = getProjectDetail(href2) //抓取项目页面
println i+ " 预售证页面 抓取项目页面"
if(projectMap){
def projList = projectMap.projList
for(def pro : projList){
def nowsellrooms = pro.yusell_roomDetails
for(def yusell : nowsellrooms){
def row = []
row << yusell["楼层"]
row << yusell["房号"]
row << yusell["座号"]
row << yusell["房号状态"]
row << pro["楼名"]
row << sellObjMap['许可证号']
row << sellObjMap['项目名称']
row << sellObjMap['所在位置']
row << sellObjMap['发展商']
row << sellObjMap['发证日期 (开始销售日期)']
row << projectMap['价款监管机构1']
row << projectMap['账户名称']
row << projectMap['账号']
row << yusell["用途"]
row << ""
row << yusell["预售-建筑面积"]
row << yusell["预售-套内建筑面积"]
row << yusell["预售-分摊面积"]
row << yusell["竣工-建筑面积"]
row << yusell["竣工-套内建筑面积"]
row << yusell["竣工-分摊面积"]
row << "深圳"
row << yusell["拟售价格-按建筑面积计"]
row << yusell["拟售价格-按套内建筑面积计"]
row << projectMap['预售总套数']
row << projectMap['现售总套数']
row << sellObjMap["栋数"]
results << row
}
}
}
}
}
ExcelUtils.write("D:\\数据\\新房\\${name}.xlsx", results, "深圳")
}
def getSellCodeObject(url){
String content = new HttpRequest().get(url, null, "utf-8")
def doc = Jsoup.parse(content);
Elements trs = doc.select("table tr")
def sellObjMap = [:]
def sellUseList = []
for(def i=1 ;i<trs.size();i++){
def tds = trs[i].select("td")
if(tds.size()==4){
sellObjMap.put(tds[0].text(), tds[1].text())
sellObjMap.put(tds[2].text(), tds[3].text())
}else if(tds.size()==6){
if(tds[1].text()!="--"){
def map = [:]
map.put(tds[0].text(), tds[1].text())
map.put(tds[2].text(), tds[3].text())
map.put(tds[4].text(), tds[5].text())
sellUseList << map
}
}else if(tds.size()==2){
sellObjMap.put(tds[0].text(), tds[1].text())
}
}
sellObjMap.useList = sellUseList
return sellObjMap;
}
def getProjectDetail(url){
String content = new HttpRequest().get(url, null, "utf-8")
def doc = Jsoup.parse(content);
// def sellInfoUrl = doc.select("strong:contains(卖方信息) a").attr("href")
// def devMap = getSellInfo("https://zjj.sz.gov.cn/ris/bol/szfdc/"+sellInfoUrl)
Elements tableEle = doc.select("table")
Elements trs1 = tableEle.get(0).select("tr")
def projMap = [:]
for(def i=1 ;i<trs1.size();i++) {
def tds = trs1[i].select("td")
for (def j = 0; j+2 <= tds.size(); j=j+2) {
projMap.put(tds[j].text(), tds[j + 1].text())
}
}
Elements trs2 = tableEle.get(1).select("tr")
def projList = []
for(def i=1 ;i<trs2.size();i++) {
def tds = trs2[i].select("td")
def map = [:]
map.put("项目名称", tds[0].text())
map.put("楼名", tds[1].text())
map.put("建设工程规划许可证", tds[2].text())
map.put("建筑工程施工许可证", tds[3].text())
def roomUrl = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+tds[4].select("a").attr("href").replace("&amp;","&")
map.put("套房信息", roomUrl)
def yusellroomList = getYuSellRoomDetail(roomUrl)
map.yusell_roomDetails = yusellroomList
// def nowsellroomList = getNowSellRoomDetail(roomUrl)
// map.nowsellroomDetails = nowsellroomList
projList << map
}
projMap.projList = projList
// projMap.devInfoMap = devMap
return projMap
}
def getYuSellRoomDetail(roomUrl){
String content = new HttpRequest().get(roomUrl, null, "utf-8")
def doc = Jsoup.parse(content);
def roomList = []
// Elements tdsEle = doc.select("td:contains(房号:)")
// for(def td : tdsEle){
// def tdText = td.text()
// if(tdText.contains("房号")){
// def roomMap = [:]
// def divEle = td.select("div")
// roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
// roomMap.put("房号状态", divEle.get(1).text())
// def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+divEle.get(1).select("a").attr("href")
// roomMap.put("房号url", href)
// roomMap = getRoomRefDetail(href, roomMap)
// roomList << roomMap
// }
// }
def hrefs = doc.select("#divShowBranch a")
for(def hrefObj : hrefs){
def danyuan = hrefObj.text().replace("[", "").replace("]", "")
def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/" + hrefObj.attr("href")
def danyuan_list = getDanyuanPage(href)
roomList.addAll(danyuan_list)
}
return roomList
}
def getDanyuanPage(roomUrl){
String content = new HttpRequest().get(roomUrl, null, "utf-8")
def doc = Jsoup.parse(content);
Elements tdsEle = doc.select("td:contains(房号:)")
def roomList = []
for(def td : tdsEle){
def tdText = td.text()
if(tdText.contains("房号")){
def roomMap = [:]
def divEle = td.select("div")
roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
roomMap.put("房号状态", divEle.get(1).text())
def href = divEle.get(1).select("a").attr("href")
if(href){
href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+ href
roomMap.put("房号url", href)
roomMap = getRoomRefDetail(href, roomMap)
}
roomList << roomMap
}
}
return roomList
}
def getContent(href){
String content = null
def error = 0
while (error<=5){
try{
content = new HttpRequest().get(href, null, "utf-8")
if(content){
error = 100
}else{
Thread.sleep(3000)
error ++
}
}catch(Exception e){
println e.getMessage() + " "+ href
Thread.sleep(3000)
error ++
}
}
return content
}
def getRoomRefDetail(href, infoMap){
String content = getContent(href)
def doc = Jsoup.parse(content);
Elements trsEle = doc.select("table tr")
println href +" "+trsEle.size()
if(trsEle.size()==0){
def error = 0
while (error<=3){
Thread.sleep(3000)
content = new HttpRequest().get(href, null, "utf-8")
doc = Jsoup.parse(content);
trsEle = doc.select("table tr")
println href +" "+trsEle.size()
if(trsEle.size()>0){
error = 100
}else{
error ++
}
}
}
for(def i=0;i<=1;i++){
def tds = trsEle[i].select("td")
for (def j = 0; j+2 <= tds.size(); j=j+2) {
infoMap.put(tds[j].text(), tds[j + 1].text())
}
}
def tds1 = trsEle.get(2).select("td")
infoMap.put("拟售价格-按建筑面积计", tds1[1].text().replace("--元/平方米","").replace("(按建筑面积计)",""))
infoMap.put("拟售价格-按套内建筑面积计", tds1[2].text().replace("--元/平方米","").replace("(按套内建筑面积计)",""))
infoMap.put("合同号", tds1[4].text())
def tds2 = trsEle.get(4).select("td")
for (def j = 0; j+2 <= tds2.size(); j=j+2) {
infoMap.put("预售-"+tds2[j].text(), tds2[j + 1].text().replace("平方米",""))
}
def tds3 = trsEle.get(6).select("td")
for (def j = 0; j+2 <= tds3.size(); j=j+2) {
infoMap.put("竣工-"+tds3[j].text(), tds3[j + 1].text().replace("平方米",""))
}
return infoMap;
}
def getSellInfo(url){
String content = new HttpRequest().get(url, null, "utf-8")
def doc = Jsoup.parse(content);
Elements trs1 = doc.select("table tr")
def devMap = [:]
for(def i=1 ;i<trs1.size();i++) {
def tds = trs1[i].select("td")
for (def j = 0; j+2 <= tds.size(); j=j+2) {
devMap.put(tds[j].text(), tds[j + 1].text())
}
}
return devMap
}
def getNowSellRoomDetail(url){
// def doc = Jsoup.parse(content);
// def header = [:]
// def VIEWSTATE = doc.select("#__VIEWSTATE").attr("value")
// def VIEWSTATEGENERATOR = doc.select("#__VIEWSTATEGENERATOR").attr("value")
// def EVENTVALIDATION = doc.select("#__EVENTVALIDATION").attr("value")
// doc.select("#__VIEWSTATEGENERATOR").attr("value")
// header.put("__VIEWSTATE", doc.select("#__VIEWSTATE").attr("value"))
// header.put("__EVENTTARGET", "imgBt2")
// header.put("__VIEWSTATEGENERATOR", doc.select("#__VIEWSTATEGENERATOR").attr("value"))
// header.put("__EVENTVALIDATION", doc.select("#__EVENTVALIDATION").attr("value"))
//// params.put("scriptManager1","updatepanel1|imgBt2")
//// String res = InvokeHttpRequest.post(url, params, "utf-8");
// getRoomNowSellTabInfo(url, VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
//
// def params = [:]
// params.put("id", "49636")
// params.put("presellid", "129765")
// HttpRequest re = new HttpRequest();
// String res = re.post(url, params, header, "utf-8");
String res = new HttpRequest().get(url+"&Branch=未命名&isBlock=xs", null, "utf-8")
def doc1 = Jsoup.parse(res);
Elements tdsEle = doc1.select("td:contains(房号:)")
def roomList = []
for(def td : tdsEle){
def tdText = td.text()
if(tdText.contains("房号")){
def roomMap = [:]
def divEle = td.select("div")
roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
roomMap.put("房号状态", divEle.get(1).text())
def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+divEle.get(1).select("a").attr("href")
roomMap.put("房号url", href)
roomMap = getRoomRefDetail(href, roomMap)
roomList << roomMap
}
}
return roomList
}
def getRoomNowSellTabInfo(targetUrl, VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION){
try {
// 创建连接
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
// 基础配置
conn.setRequestMethod("POST");
conn.setDoOutput(true); // 允许写入输出流
conn.setConnectTimeout(5000); // 5秒连接超时
conn.setReadTimeout(10000); // 10秒读取超时
// 设置请求头(模拟浏览器)
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9");
// 构建Form Data参数(带中文需编码)
String formData = "__VIEWSTATE=" + URLEncoder.encode(VIEWSTATE, "UTF-8") +
"&__VIEWSTATEGENERATOR=" + URLEncoder.encode(VIEWSTATEGENERATOR, "UTF-8") +
"&__EVENTVALIDATION="+URLEncoder.encode(EVENTVALIDATION, "UTF-8") +
"&__EVENTTARGET=imgBt2";
// 发送请求体
OutputStream os = conn.getOutputStream()
byte[] input = formData.getBytes("utf-8");
os.write(input, 0, input.length);
// 获取响应状态码
int status = conn.getResponseCode();
System.out.println("响应状态码: " + status);
String responseLine;
// 读取响应内容
if (status == HttpURLConnection.HTTP_OK) {
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"))
StringBuilder response = new StringBuilder();
while ((responseLine = br.readLine()) != null) {
response.append(responseLine.trim());
}
System.out.println("响应内容:\n" + response.toString());
} else {
System.out.println("请求失败,错误码: " + status);
}
return responseLine
} catch (Exception e) {
e.printStackTrace();
}
return null
}
}
@@ -3,10 +3,12 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang3.SerializationUtils
import java.util.regex.Pattern
@@ -14,15 +16,30 @@ class AddrAlaisProcess {
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
def fullCityMap = CommonDatautils.queryFullNameCity()
//
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
def task = new AddrAlaisProcess()
// task.run()
// task.splitDist()
// task.addAddr()
task.nineApi()
// task.nineApi()
// task.createResult()
// task.mergeResult()
// task.addLocation()
task.updateAddr()
}
def addLocation(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\地图无路.xlsx",{d->
def obj = CommonDatautils.findDistrictById(d['A'])
if(obj){
d['G'] = obj.lng/10000000000.0
d['H'] = obj.lat/10000000000.0
}
})
}
def splitDist(){
@@ -102,7 +119,7 @@ class AddrAlaisProcess {
def nineApi(){
def i = 0;
while(i<20) {
while(i<1) {
def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx"
@@ -122,6 +139,16 @@ class AddrAlaisProcess {
def roadObj = findBaiduDatas( data.city, d['H'])
if(roadObj){
d['J'] = "百度存在"
}else{
// def distData = mcaDistMap.get(d['D'] + "#" + d['E'])
// def fullDistName = null
// if (distData) {
// fullDistName = distData.district
// }
// def obj = CommonDatautils.findGaoDeDatas(data.city, fullDistName, d['H'])
// if(obj){
// d['J'] = "高德存在"
// }
}
}
}
@@ -208,4 +235,39 @@ class AddrAlaisProcess {
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
def updateAddr(){
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\地址及地址别名-详情-1.xlsx",{d->
def obj = CommonDatautils.findDistrictById(d['C'])
if(obj && d['G']){
def flag = DataHandleUtils.isStandardRoad(d['L'])
def udata = new BasicDBObject()
if(obj.address == d['H']){
udata.put("address", d['L'])
}else{
if(obj.addr_alias){
def new_addr_alias = SerializationUtils.clone(obj.addr_alias)
for(def alias : new_addr_alias){
if(d['H'] == alias){
if(flag){
if(d['L']!=alias){
alias = d['L']
}
}else{
alias = ""
}
}
}
if(new_addr_alias!=obj.addr_alias){
udata.put("addr_alias", new_addr_alias)
}
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', obj._id),new BasicDBObject('$set',udata))
}
}
})
}
}
@@ -225,15 +225,12 @@ class AutoAddOfficeBatch extends BaseBatch {
}
def repeatDist(){
def citys = ["南京"]//CommonDatautils.findAllCity().reverse()
def citys = ["成都"]//CommonDatautils.findAllCity().reverse()
def allrets = []
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
def index = 0
for (def c : citys) {
index++
if(["宁波","无锡","青岛"].contains(c)){
continue
}
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
def rets = checkRepeatDistrictBatch(c, districts)
println c + " " + districts.size()
@@ -254,13 +251,14 @@ class AutoAddOfficeBatch extends BaseBatch {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityId = cityCodeDataMap.get(city)
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502)
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202505)
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def total = districts.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for (def i = 0; i < total; i++) {
println i
final def o1 = districts[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
@@ -0,0 +1,86 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CommunityShapeProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
static main(args) {
def task = new CommunityShapeProcess();
task.run();
}
def run(){
def q = new BasicDBObject()
q.put("province_name", "河北省")
q.put("shape", new BasicDBObject('$exists', true))
def keys = new BasicDBObject()
keys.put("shape", 1)
def dbcursor = community_CONN.find(q, keys).batchSize(1000)
for(def d : dbcursor){
if(d.shape.type=="Polygon"){
def coordinates = d.shape.coordinates.get(0)
def points = []
def index = 0
for(def arrs : coordinates){
if(index!=0 && index+1!= coordinates.size()){
if(!points.contains(arrs)){
points << arrs
}else{
println d._id +" 重复数据:"+arrs
}
}else{
points << arrs
}
index++
}
if(points.size()!=coordinates.size()){
d.shape.coordinates = [points]
def udata = new BasicDBObject()
udata.shape = d.shape
community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}else{
println d._id
def coordinateArrs = d.shape.coordinates
def multiPoints = []
def updateFlag = false
for(def typeCoordinates : coordinateArrs){
def pointArrs = []
for(def coordinates : typeCoordinates){
def points = []
def index = 0
for(def arrs : coordinates){
if(index!=0 && index+1!= coordinates.size()){
if(!points.contains(arrs)){
points << arrs
}else{
println d._id +" 重复数据:"+arrs
updateFlag = true
}
}else{
points << arrs
}
index++
}
pointArrs << points
}
multiPoints << pointArrs
}
if(updateFlag){
d.shape.coordinates = multiPoints
def udata = new BasicDBObject()
udata.shape = d.shape
community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
}
}
@@ -93,12 +93,23 @@ class DevProcess {
}
def aiDevTown(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
def message = res.data.replaceAll("\n", "")
d['F'] = res.code
d['G'] = message
println d['E'] +" "+res.code + " "+message
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
// def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
// def message = res.data.replaceAll("\n", "")
// d['F'] = res.code
// d['G'] = message
// println d['E'] +" "+res.code + " "+message
def q = new BasicDBObject("name", d['E']).append("type","乡镇街道")
def dbcursor = DEV_MAPPING_CONN.find(q)
if(dbcursor.count()>0){
def list = []
for(def db : dbcursor){
list << db.township
}
d['H']= list.join(",")
}
})
}
@@ -0,0 +1,75 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.looplineinfo.InPolygon
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.awt.geom.Point2D
class DistrictLoopInfo {
DBCollection LATLNG_POLYGON_COLL = MongoUtils.getCollectionByName("fangjia", "latlng_polygon", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictLoopInfo()
task.run()
}
def run(){
def q = new BasicDBObject()
q.put("name", new BasicDBObject('$in', ["内环","中环","外环","郊环"]))
def loopShapes = LATLNG_POLYGON_COLL.find(q)
def loopMap = [:]
for(def loop : loopShapes){
loopMap.put(loop.name, loop)
}
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city", "region", "name", "lat", "lng", "loop_line_info"])
println districts.size()
def index = 0
for(def d : districts){
index++
if(d.lng){
def lng = d.lng/10000000000.0
def lat = d.lat/10000000000.0
Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6, lng), DataHandleUtils.getDoubleRound(6, lat));
def loop1 = loopMap.get("内环")
def flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
def loop_line_info = ""
if(flag){
loop_line_info = "内环内"
}else{
loop1 = loopMap.get("中环")
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
if(flag){
loop_line_info = "内中环"
}else{
loop1 = loopMap.get("外环")
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
if(flag){
loop_line_info = "中外环"
}else{
loop1 = loopMap.get("郊环")
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
if(flag){
loop_line_info = "外郊环"
}else{
loop_line_info = "郊环外"
}
}
}
}
println index +" "+d.region +" "+d.name+" 原:"+d.loop_line_info+" 现:"+loop_line_info+" lng:"+lng+","+lat
def udata =new BasicDBObject()
udata.put("loop_line_info", loop_line_info)
DBUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set',udata))
}else{
println index + " "+d.region +" "+d.name+"无经纬度信息"
}
}
}
}
@@ -22,7 +22,47 @@ class DistrictRepeatBatch {
static main(args) {
def task = new DistrictRepeatBatch()
// task.run()
task.resultProcess()
// task.resultProcess()
task.processRepeatAndAlias()
}
def processRepeatAndAlias(){
ExcelUtils.modify("D:\\数据\\小区\\重复小区\\结果\\重复可疑小区-南京.xlsx",{d->
if(d['H'] && d['H'] == "删除"){
def obj = CommonDatautils.findDistrictById(d['A'])
if(!obj){
obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
}
if(obj){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
saveOptLog(obj.city, obj.region, obj.name, "remove")
}
}else{
if(d['I']){
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(obj){
def arrs = d['I'].split(",")
if(arrs){
def alias = obj.alias
if(!alias){
alias = []
}
alias.addAll(arrs)
alias = alias.unique()
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
saveOptLog(obj.city, obj.region, obj.name, "update")
}
}
}
}
},0)
}
def run(){
@@ -17,7 +17,7 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class GXDdistrictMatch {
class 8GXDdistrictMatch {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -29,8 +29,8 @@ class TownMiddleProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping_0526", MONGO100_URL)
DBCollection TEMP_ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
@@ -65,7 +65,7 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
synchTown()
// synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
@@ -101,11 +101,22 @@ class TownMiddleProcess {
// updateNewRegionPoyline()//更新新的区域围栏
updateMappingPolyline()
// updateMappingPolyline()
//批量修改多边型围栏数据结构
// updatePolylineBatch()
// test2dshape()
//将省会城市更新到数据库中
// updateProvinceCenter()
//城市级别更新到库中
// updateCityLevel()
//城市顺序
// updateCityNum()
}
def createProvince(){
def cat = "省"
def tjjMaps = findAllProcincesFromTJJ(cat)
@@ -457,16 +468,17 @@ class TownMiddleProcess {
def datas = []
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
if (datas.size() >= 5000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
// ADMINISTRATIVE_MAPPING_COLL.insert(datas)
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_COLL.insert(datas)
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear();
}
@@ -1449,4 +1461,123 @@ class TownMiddleProcess {
}
def updatePolylineBatch(){
def q = new BasicDBObject()
q.put("shape",new BasicDBObject('$exists', true))
q.put("update_shape",new BasicDBObject('$exists', false))
def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100)
for(def d : dbcursor){
def type = d.shape.type
def udata = new BasicDBObject()
def shapeMap = [:]
if(type == "polygon"){
shapeMap.put("type", "Polygon")
shapeMap.put("coordinates", d.shape.coordinates)
udata.put("shape", shapeMap)
udata.put("update_shape", 1)
ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata))
}else if(type == "MultiPolygon"){
def coordinates = d.shape.coordinates
def new_coors = []
for(def coors : coordinates){
new_coors << [coors]
}
shapeMap.put("type", "MultiPolygon")
shapeMap.put("coordinates", new_coors)
udata.put("shape", shapeMap)
udata.put("update_shape", 1)
ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata))
}
}
}
def test2dshape(){
def q = new BasicDBObject()
q.put("street","花亭路街道")
// q.put("shape",new BasicDBObject('$exists', true))
def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100)
for(def obj : dbcursor){
try{
TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj)
}catch(Exception e){
def coordinates = obj.shape.coordinates
for(def coors : coordinates){
for (int i = 0; i < coors.size(); i++) { // 改为索引遍历
def coorsArr = coors[i]
def index = 0
def new_coors = []
for(def points : coorsArr){
index ++
if(!new_coors.contains(points)){
new_coors << points
}else{
if(index<=coorsArr.size()){
new_coors << points
break;
}
}
}
if(new_coors.size()!=coorsArr.size()){
coors[i] = new_coors
println "[修改] "+ obj.city +" "+obj.street +" 原:"+coorsArr.size()+" 现:"+new_coors.size()
}
}
}
try{
TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj)
}catch(Exception ex){
println "仍有问题:"+obj.city +" "+obj.street+" "+ex.getMessage()
}
}
}
}
def updateProvinceCenter(){
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\省会城市.xlsx",{d->
def city = d['C']
if(city){
def q = new BasicDBObject("type","城市").append("city_short", city)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject("capital_city", "是")
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
}
}
})
}
def updateCityLevel(){
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市等级.xlsx",{d->
def city = d['A']
if(city){
def q = new BasicDBObject("type","城市").append("city_short", city)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject("city_level", d['B'])
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
}else{
println city +" 没找到"
}
}
})
}
def updateCityNum(){
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市排序.xlsx",{d->
def city = d['A']
if(city){
def q = new BasicDBObject("type","城市").append("city_short", city)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject("city_position", d['B'].toString().toDouble().toInteger())
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
}else{
println city +" 没找到"
}
}
})
}
}
@@ -18,16 +18,17 @@ class GaodeCityRange {
}
def run(){
def citys = CommonDatautils.findAllCity()
// def citys = CommonDatautils.findAllCity()
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
def citys = ["新星","胡杨河","白杨"]
HttpRequest request = new HttpRequest()
def rows = []
citys.each{c->
if(!citys1.contains(c)){
c = c.replace("州","")
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=66e6771767ab293ace441553c36621c1&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
if(resp.containsKey("districts")){
def datas = resp.get("districts")
for(def d : datas){
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.location
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
@@ -22,6 +23,11 @@ class RegionPointCrawl {
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static DBCollection TOWN_CENTERCOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_center")
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionPointCrawl();
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
@@ -40,8 +46,12 @@ class RegionPointCrawl {
// println "区域围栏 完成"
// }
task.statSame()
// task.statSame()
//"福建省","河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","山东省","湖北省","湖南省","江西省","江苏省","安徽省"
def provinces = ["河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","青海省","山东省","福建省","浙江省","台湾省","河南省","湖北省","湖南省","江西省","江苏省","安徽省","广东省","海南省","四川省","贵州省","云南省","北京市","上海市","天津市","重庆市","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
task.getTownshipCenter(provinces)
task.addTownCenter(provinces)
}
def crawlCityAndDistrictCode(def province){
@@ -210,7 +220,7 @@ class RegionPointCrawl {
jo = JSONObject.fromObject(html)
if(jo.infocode!="10000"){
errorCount ++
Thread.sleep(30*1000)
Thread.sleep(3*1000)
// println " 不可用Key: "+key
}else{
errorCount = 100
@@ -228,15 +238,15 @@ class RegionPointCrawl {
static String getRandomKey2() {
def KeyList = [
"2af0b8533415a612b98d59bb93d664b9",
"0246c57bbcf24aa246c207b3c99dfe41",
"042328b3f0fa5daf426802d379521d87",
"055d4f1b5b64a2bce18f4fe1f15c2639",
"d5484f5170502405d88bbaffe1b496f0",
"e74ccc4258079d2200199695110cceb1",
"f58629fc799fbc8537ca85046c6d44a5",
"291073f17ee0b963ccb1927ed92bf265",
"2a1fb75990f2b39f6221e50892dd3988"
// "2af0b8533415a612b98d59bb93d664b9",
// "0246c57bbcf24aa246c207b3c99dfe41",
// "042328b3f0fa5daf426802d379521d87",
// "055d4f1b5b64a2bce18f4fe1f15c2639",
// "d5484f5170502405d88bbaffe1b496f0",
// "e74ccc4258079d2200199695110cceb1",
// "f58629fc799fbc8537ca85046c6d44a5",
"eb69e4ad2a9cdd31f163a0ed32effe6e"
// "2af0b8533415a612b98d59bb93d664b9"
]
def count = RandomUtils.nextInt(KeyList.size())
@@ -300,4 +310,117 @@ class RegionPointCrawl {
}
return map;
}
def getTownshipCenter(provinces){
for(def p : provinces){
def query = new BasicDBObject("province", p)
def count = TOWN_CENTERCOLL.count(query)
if(count){
println p +" 已存在"
}else{
def ret = polygon(p, "3", "base");
if(ret){
def citys = ret[0].districts
if(citys){
def allRets = []
for(def c : citys){
def cityTowns = getTownByCity(p, c.name,c.districts)
allRets.addAll(cityTowns)
}
saveTownCenter(allRets)
println p +" 保存:"+allRets.size()
}
}else{
println p+ "无结果"
}
}
}
}
def getTownByCity(prov, city,distDatas){
def rets = []
for(def d : distDatas){
def townDatas = d.districts
for(def town : townDatas){
town.province = prov
town.district = d.name
town.city = city
rets << town
}
}
return rets
}
def saveTownCenter(allRets){
for(def r : allRets){
def obj = new BasicDBObject()
obj.put("province", r.province)
obj.put("city", r.city)
obj.put("district", r.district)
obj.put("town", r.name)
obj.put("center", r.center)
obj.put("level", r.level)
TOWN_CENTERCOLL.save(obj)
}
}
def addTownCenter(provinces){
for(def p : provinces){
def q = new BasicDBObject("province",p).append("type","乡镇街道")
// q.put("location", new BasicDBObject('$exists', false))
def dbcursor = administrative_mapping_coll.find(q)
for(def obj : dbcursor){
def tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", obj.street)
def dbo = TOWN_CENTERCOLL.findOne(tq)
if(!dbo){
def new_street = obj.street.replace("管理处","")
tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", new_street)
dbo = TOWN_CENTERCOLL.findOne(tq)
}
if(dbo){
def arrs = dbo.center.split(",")
def location = [:]
location["lon"] = arrs[0].toDouble()
location["lat"] = arrs[1].toDouble()
if(!obj.location){
def udata = new BasicDBObject("location", location)
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
println("【修改】"+obj.city +" "+ obj.district+" "+obj.street)
}else{
def distance = com.fangjia.script.util.GeoUtils.calculateDistance(location.lat, location.lon, obj.location.lat, obj.location.lon);
if(distance>1000){
def udata = new BasicDBObject("location", location)
println(obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 经纬度中心点相距1公里以外 原:"+obj.location.lon+","+obj.location.lat+" 新:"+dbo.center+" 距离:"+distance)
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
}
}else{
def esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street)
if(!esobj){
def short_prov = obj.province.replace("省", "")
if(obj.street.contains(short_prov)){
esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street.replace(short_prov, ""))
}
}
if(esobj){
def location = [:]
location["lon"] = esobj.location.lon.toDouble()
location["lat"] = esobj.location.lat.toDouble()
def udata = new BasicDBObject("location", location)
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
println("【修改】"+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street+" "+ esobj.cat+" "+esobj.name +" es poi ")
}else{
println(obj.province+" "+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 没有找到")
}
}
}
}
}
}
@@ -0,0 +1,15 @@
package com.fangjia.script.gj.test
class NonghangDataExport {
static main(args) {
def task = new NonghangDataExport();
task.run()
}
def run(){
def citys = ["ÁÙÒÊ","¹ãÖÝ", "ÉîÛÚ"]
for(def c : citys){
}
}
}
@@ -2,6 +2,7 @@ package com.fangjia.script.linxsh.util
import com.fangjia.core.RegionData
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBCursor
@@ -47,6 +48,8 @@ class AddressUtil {
private static final cityMap = new HashMap<String, String>()
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
static {
initCityMap()
}
@@ -55,9 +58,7 @@ class AddressUtil {
* 初始化城市数据
*/
static def initCityMap() {
Mongo mongo = new Mongo("mongo.db.fangjia.com", 28017);
DBCollection SERVER_AREA = mongo.getDB("fangjia").getCollection("server_area");
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1));
DBCursor cursor = SERVER_AREA_COLL.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1));
for(DBObject dbObject : cursor) {
if(!"全国".equals(dbObject.display)) {
cityMap.put(dbObject.display, dbObject.display)
@@ -239,7 +239,7 @@ public class CommonUtils {
println(cmd)
DBObject data = new BasicDBObject()
data.put("category", "sell")
data.put("priority", 3)
data.put("priority", 1)
data.put("status", 0)
data.put("need_redraw_trend", needRedrawTrend)
data.put("remark", 'week')
@@ -1548,6 +1548,34 @@ public class OpenApiClient {
}
public static JSONObject houseAction(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/house/around", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/house/around", params, "utf-8")
println("${city} ${address} ³ÌÐòµ÷ÓÃÒì³£")
Thread.sleep(6000)
}
if (result.getInt("code") == 0) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
}
@@ -21,15 +21,15 @@ public class DistrictMonthlyCalc{
public static void main(String[] args) {
List<String> cities = ["ÉϺ£"];
def allCity = CommonUtils.findAllCity()
cities.addAll(allCity)
List<String> cities = ["ÏåÑô"];
// def allCity = CommonUtils.findAllCity()
// cities.addAll(allCity)
def districts = []
cities.each {def city->
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("name", 1).append("region", 1)).batchSize(2000).toList()
println(city)
}
ExecutorService executorService = Executors.newFixedThreadPool(20)
ExecutorService executorService = Executors.newFixedThreadPool(1)
districts.each {BasicDBObject district->
executorService.submit(new Calculator(district))
}
@@ -43,7 +43,7 @@ public class DistrictMonthlyCalc{
class Calculator extends Thread{
static DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat_new", "mongo.db.fangjia.com")
final BasicDBObject district;
public Calculator(final BasicDBObject district){
@@ -52,14 +52,22 @@ class Calculator extends Thread{
@Override
void run() {
REGION_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo->
def q = new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)
q.put("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01", "yyyy-MM-dd")))
REGION_STAT.find(q).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo->
Date weekend = dbo.weekend
if(DateUtil.isLastSundayOfMonth(weekend)){
def trend = new BasicDBObject("city", district.city)
trend.append("region", district.region)
trend.append("district_name", district.name)
trend.append("date", weekend.format("yyyyMM").toInteger())
REGION_STAT_TREND.update(trend , new BasicDBObject('$set', new BasicDBObject("avg_price", dbo.avg_price)),true,false)
def udata = new BasicDBObject()
if(dbo.avg_price_2b && dbo.value_type!=1){
udata.put("avg_price", dbo.avg_price_2b)
}else{
udata.put("avg_price", dbo.avg_price)
}
REGION_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
}
}
REGION_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("data", 1)).each {BasicDBObject trend->