diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy index 1d1396141..cedebcd9f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy @@ -29,7 +29,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch { @Override void process(Object args) { List cities = []; - def allCity = CommonUtils.findAllCity() + def allCity = ["广州","深圳","临沂"]//CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) cities.addAll(allCity) @@ -38,7 +38,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch { districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList() println(city) } - ExecutorService executorService = Executors.newFixedThreadPool(20) + ExecutorService executorService = Executors.newFixedThreadPool(1) districts.each {BasicDBObject district-> executorService.submit(new Calculator(district)) } @@ -65,22 +65,26 @@ class Calculator extends Thread{ } def statMap = [:] - DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo-> + DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type).append("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01","yyyy-MM-dd")))).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo-> Date weekend = dbo.weekend if(DateUtil.isLastSundayOfMonth(weekend)){ def trend = new BasicDBObject("city", district.city) trend.append("region", district.region) trend.append("district_name", district.name) - trend.append("estate_type", estate_type) +// trend.append("estate_type", estate_type) trend.append("date", weekend.format("yyyyMM").toInteger()) def udata = new BasicDBObject(); - udata.put("avg_price", dbo.avg_price) - udata.put("credit", dbo.credit) - udata.put("m_date", new Date()) - DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false) - statMap.put(weekend.format("yyyyMM").toInteger(), udata) + if(dbo.avg_price_2b){ +// udata.put("avg_price", dbo.avg_price) + udata.put("avg_price", dbo.avg_price_2b) + udata.put("credit", dbo.credit) + udata.put("m_date", new Date()) + DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false) + statMap.put(weekend.format("yyyyMM").toInteger(), udata) + } + } } DISTRICT_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).sort(new BasicDBObject("date", 1)).each {BasicDBObject trend-> @@ -88,14 +92,14 @@ class Calculator extends Thread{ def monDate = getMomDate(date) def yoyDate = getYoyDate(date) BasicDBObject monDbo = statMap.get(monDate) - if(monDbo != null && monDbo.avg_price != 0) { - trend.mom = DataUtil.roundDouble(((trend.avg_price - monDbo.avg_price)/(double)monDbo.avg_price),4) + if(monDbo != null && monDbo.avg_price_2b != 0) { + trend.mom = DataUtil.roundDouble(((trend.avg_price_2b - monDbo.avg_price_2b)/(double)monDbo.avg_price_2b),4) }else{ trend.mom = 0d } BasicDBObject yoyDbo = statMap.get(yoyDate) if(yoyDbo != null && yoyDbo.avg_price != 0){ - trend.yoy = DataUtil.roundDouble(((trend.avg_price - yoyDbo.avg_price)/(double)yoyDbo.avg_price),4) + trend.yoy = DataUtil.roundDouble(((trend.avg_price_2b - yoyDbo.avg_price_2b)/(double)yoyDbo.avg_price_2b),4) }else{ trend.yoy = 0d } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index d96e0da05..88eb4cbd3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年5月深圳成交明细.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index d18bc364e..c6a241b54 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -57,9 +57,9 @@ class DistrictPriceExportBatch extends BaseBatch{ } def sb = appendResult(headers, results) DataHandleUtils.outPutFile(fileName, sb, charset); - FzFtpUtils.ftpUpload(fileName) - //生成加密数据文件并上传 - FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset); +// FzFtpUtils.ftpUpload(fileName) +// //生成加密数据文件并上传 +// FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset); } def selectLastResultDatas(def rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 5a6cba7f2..09de965da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.fuzhou +import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner @@ -63,7 +64,7 @@ class DistrictSellStatBatch extends BaseBatch { } def calcMonthlyPrice(){ - //只有福州使用tob价格 + //只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外 def citys = CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index d049deab9..8ebd96798 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -82,7 +82,7 @@ class FzDataResultStat { } // List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) - def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"))) + def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -4, "yyyyMM"))) def map2 = [:] for(def obj : list2) { def key = obj.city + "##" + obj.region + "##" + obj.name @@ -119,7 +119,7 @@ class FzDataResultStat { index++ println row } - ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") + ExcelUtils.write("D:\\数据\\福州\\价格比对4_${curMonth}.xlsx", rows, "数据") def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index cb2fcce9c..cbeb06541 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -13,11 +13,17 @@ import com.mongodb.DBCursor import com.mongodb.DBObject import org.apache.solr.common.util.DateUtil +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + class AddAvgpriceBatch { private static def citys = null; private static def districtMap = [:]; public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100) static { citys = ["宜昌"]//CommonDatautils.findAllCity() @@ -26,10 +32,107 @@ class AddAvgpriceBatch { static main(args) { def task = new AddAvgpriceBatch(); - task.run1(); +// task.run1(); // task.test() // task.sendPriceCommand() // task.sendNewDistPrice() + +// task.delNoExistsDistPrice() + task.checkToBprice() + } + + def checkToBprice(){ + def citys = CommonDatautils.findAllCity() + for(def i=0;i + + def districts = null + if(d['D']=="义乌"){ + districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"]) + def newDists = [] + for(def dist : districts){ + if(dist.region =="义乌"){ + newDists << dist + } + } + districts = newDists + }else if(d['D']=="金华"){ + districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"]) + def newDists = [] + for(def dist : districts){ + if(dist.region !="义乌"){ + newDists << dist + } + } + districts = newDists + }else if(d['D']=="无锡"){ + districts = CommonDatautils.findAllDistrictByCity("无锡", "district", ["region","name"]) + def newDists = [] + for(def dist : districts){ + if(dist.region !="江阴"){ + newDists << dist + } + } + districts = newDists + }else if(d['D']=="苏州"){ + districts = CommonDatautils.findAllDistrictByCity("苏州", "district", ["region","name"]) + def newDists = [] + for(def dist : districts){ + if(dist.region !="常熟"){ + newDists << dist + } + } + districts = newDists + }else if(d['D']=="济南"){ + districts = CommonDatautils.findAllDistrictByCity("济南", "district", ["region","name"]) + def newDists = [] + for(def dist : districts){ + if(dist.region !="莱芜"){ + newDists << dist + } + } + districts = newDists + }else{ + districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["region","name"]) + } + + d['F'] = districts.size() + if(districts){ + d['E'] = "是" + }else{ + d['E'] = "否" + } + + def prov = null + if(districts){ + index ++ + def data = fullCityMap.get(d['D']) + if(data){ + prov = data.province + } + def str = districts.collect { it.name }.join(", ") + rets << ["id":index, "type":"A", "province":prov, "city":d['D'], "names":str] + + createDoc(rets) + } + }, 0) + +// FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表1.dat", [], "utf-8", false); + } + + def createDoc(dataList){ + // 创建 Word 文档 + def doc = new XWPFDocument() + def table = doc.createTable(dataList.size() + 1, 5) // 行数=数据行+表头,列数=3 + + // 设置表头 + def headerRow = table.getRow(0) + + headerRow.getCell(0).setText("序号") + headerRow.getCell(1).setText("类型") + headerRow.getCell(2).setText("省份") + headerRow.getCell(3).setText("城市") + headerRow.getCell(4).setText("楼盘名称") + + // 填充数据 + dataList.eachWithIndex { item, index -> + def row = table.getRow(index+1) // 跳过表头 + row.getCell(0).setText(item.id.toString()) + row.getCell(1).setText(item.type.toString()) + row.getCell(2).setText(item.province.toString()) + row.getCell(3).setText(item.city.toString()) + row.getCell(4).setText(item.names.toString()) + } + + // 保存到文件 + def outputFile = new File("A类城市楼盘列表.docx") + doc.write(new FileOutputStream(outputFile)) + println "Word 表格生成成功: ${outputFile.absolutePath}" + } + + def cityStat(){ + + def index = 0 + def rets = [] + ExcelUtils.modify("D:\\数据\\小区\\城市数量统计.xlsx",{d-> + def districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["city", "region", "name"]) + if(districts){ + d['F'] = districts.size() + d['E'] = "是" + + rets << ["序号","类型","省份","城市","楼盘名称","金华"] + def data = fullCityMap.get(d['D']) + def prov = null + if(data){ + prov = data.province + } + + for(def dist : districts){ + index++ + rets << [index, "A", prov,dist.city, dist.name, dist.region] + } + } + }, 0) + +// ExcelUtils.write("D:\\数据\\小区\\A类城市楼盘列表.xlsx", rets, "楼盘数据详情列表") + FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表.dat", [], "utf-8", false); + } + + def stat(city){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name","alias"]) + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + def result = Collections.synchronizedList(new ArrayList()); + def countResult = Collections.synchronizedList(new ArrayList()); + def rets = [] +// def houseTotal = CommonDatautils.findHouseCountByDate(city, null, null, null, "sell", 20240611) + rets << city + rets << districts.size() +// rets << houseTotal + for(def i=0;i @@ -87,7 +89,7 @@ class FangjiatongUserStat { } rets << ret } - ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019") + ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2022") } def findUses(role){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 576869ecd..808276641 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -21,7 +21,7 @@ class FileUtils { def sb = new StringBuffer(); if(headers){ - sb.append(headers.join("|@|")) + sb.append(headers.join("@")) sb.append("\n"); } @@ -32,7 +32,7 @@ class FileUtils { for(def row : ret){ if(row){ index++ - sb.append(StringUtils.join(row.toArray(), "|@|")) + sb.append(StringUtils.join(row.toArray(), "@")) sb.append("\n"); if(index%10000==0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/SzGovHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/SzGovHouseCrawl.groovy new file mode 100644 index 000000000..e96448d1e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/SzGovHouseCrawl.groovy @@ -0,0 +1,583 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.newDistrict.InvokeHttpRequest +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject +import net.sf.json.groovy.JsonSlurper +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +class SzGovHouseCrawl { + + static main(args) { + def task = new SzGovHouseCrawl(); + task.run(); + } + + def run(){ + def index_content = """ + + + + + + 一手房源公示 + + + +
+
+ + + +
+ + + + + + + + + +
+ +\t +\t +\t +
+ +
+
+ + + + + +
+

+ 项目查询

+

+ 开发企业

+

+ 宗地地址

+ +
+ + + + +
+
+\t +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ 序号 + + 预售证号 + + 项目名称 + + 开发企业 + + 所在区 + + 批准时间 +
+ 1 + + + 深房许字(2024)罗湖002号 + + + 凌云华府 + + 深圳市城建产业园发展有限公司 + + 罗湖 + + 2024-06-27 +
+ 2 + + + 深房许字(2022)罗湖007号 + + + 高投大厦 + + 深圳市城建产业园发展有限公司 + + 罗湖 + + 2022-12-16 +
+ +
+
+
+
+ +\t +\t +\t + +\t +
+ +
+
+ +
+
+ + + + + +
+ + + + """ + + getRoomRefDetail("https://zjj.sz.gov.cn/ris/bol/szfdc/housedetail.aspx?id=1979350",[:]) + def doc = Jsoup.parse(index_content); + Elements trs = doc.select("table tr") + def name = "深圳市城建产业园发展有限公司" + def results = [] + def header = ["楼层","房间号","单元号","销售状态","楼栋名称","预售许可证号","项目名称","所属项目地址","开发商名称","开盘日期","监管银行","监管账户名称","监管账户账号","房屋用途","户型","预测总建筑面积","预测套内建筑面积","预测公摊建筑面积","实际总建筑面积","实际套内建筑面积","实际公摊建筑面积","城市","拟售价格-按建筑面积计","拟售价格-按套内建筑面积计","预售总套数","现售总套数","栋数"] + results << header + //开发商查询结果页 + println trs.size()-2 +" 预售证" + for(def i=1 ;i0){ + error = 100 + }else{ + error ++ + } + } + + } + for(def i=0;i<=1;i++){ + def tds = trsEle[i].select("td") + for (def j = 0; j+2 <= tds.size(); j=j+2) { + infoMap.put(tds[j].text(), tds[j + 1].text()) + } + } + + def tds1 = trsEle.get(2).select("td") + infoMap.put("拟售价格-按建筑面积计", tds1[1].text().replace("--元/平方米","").replace("(按建筑面积计)","")) + infoMap.put("拟售价格-按套内建筑面积计", tds1[2].text().replace("--元/平方米","").replace("(按套内建筑面积计)","")) + infoMap.put("合同号", tds1[4].text()) + + + def tds2 = trsEle.get(4).select("td") + for (def j = 0; j+2 <= tds2.size(); j=j+2) { + infoMap.put("预售-"+tds2[j].text(), tds2[j + 1].text().replace("平方米","")) + } + + def tds3 = trsEle.get(6).select("td") + for (def j = 0; j+2 <= tds3.size(); j=j+2) { + infoMap.put("竣工-"+tds3[j].text(), tds3[j + 1].text().replace("平方米","")) + } + return infoMap; + } + + def getSellInfo(url){ + String content = new HttpRequest().get(url, null, "utf-8") + def doc = Jsoup.parse(content); + Elements trs1 = doc.select("table tr") + def devMap = [:] + for(def i=1 ;i + def obj = CommonDatautils.findDistrictById(d['A']) + if(obj){ + d['G'] = obj.lng/10000000000.0 + d['H'] = obj.lat/10000000000.0 + } + }) } def splitDist(){ @@ -102,7 +119,7 @@ class AddrAlaisProcess { def nineApi(){ def i = 0; - while(i<20) { + while(i<1) { def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx" @@ -122,6 +139,16 @@ class AddrAlaisProcess { def roadObj = findBaiduDatas( data.city, d['H']) if(roadObj){ d['J'] = "百度存在" + }else{ +// def distData = mcaDistMap.get(d['D'] + "#" + d['E']) +// def fullDistName = null +// if (distData) { +// fullDistName = distData.district +// } +// def obj = CommonDatautils.findGaoDeDatas(data.city, fullDistName, d['H']) +// if(obj){ +// d['J'] = "高德存在" +// } } } } @@ -208,4 +235,39 @@ class AddrAlaisProcess { def obj = BAIDU_POI_COLL.findOne(q) return obj; } + + def updateAddr(){ + ExcelUtils.modify("D:\\数据\\数据建设\\小区\\地址及地址别名-详情-1.xlsx",{d-> + def obj = CommonDatautils.findDistrictById(d['C']) + if(obj && d['G']){ + def flag = DataHandleUtils.isStandardRoad(d['L']) + def udata = new BasicDBObject() + if(obj.address == d['H']){ + udata.put("address", d['L']) + }else{ + if(obj.addr_alias){ + def new_addr_alias = SerializationUtils.clone(obj.addr_alias) + for(def alias : new_addr_alias){ + if(d['H'] == alias){ + if(flag){ + if(d['L']!=alias){ + alias = d['L'] + } + }else{ + alias = "" + } + } + } + if(new_addr_alias!=obj.addr_alias){ + udata.put("addr_alias", new_addr_alias) + } + } + } + if(udata){ + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', obj._id),new BasicDBObject('$set',udata)) + } + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy index a35463b66..2441ae51e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy @@ -225,15 +225,12 @@ class AutoAddOfficeBatch extends BaseBatch { } def repeatDist(){ - def citys = ["南京"]//CommonDatautils.findAllCity().reverse() + def citys = ["成都"]//CommonDatautils.findAllCity().reverse() def allrets = [] allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"] def index = 0 for (def c : citys) { index++ - if(["宁波","无锡","青岛"].contains(c)){ - continue - } def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"]) def rets = checkRepeatDistrictBatch(c, districts) println c + " " + districts.size() @@ -254,13 +251,14 @@ class AutoAddOfficeBatch extends BaseBatch { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityId = cityCodeDataMap.get(city) - def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502) + def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202505) def addCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) def total = districts.size() def rets = new CopyOnWriteArrayList() def sets = new CopyOnWriteArraySet() for (def i = 0; i < total; i++) { + println i final def o1 = districts[i] // fixedThreadPool.execute(new Runnable() { // void run() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/CommunityShapeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/CommunityShapeProcess.groovy new file mode 100644 index 000000000..7292a0d0f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/CommunityShapeProcess.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CommunityShapeProcess { + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + static main(args) { + def task = new CommunityShapeProcess(); + task.run(); + } + + def run(){ + def q = new BasicDBObject() + q.put("province_name", "河北省") + q.put("shape", new BasicDBObject('$exists', true)) + def keys = new BasicDBObject() + keys.put("shape", 1) + def dbcursor = community_CONN.find(q, keys).batchSize(1000) + for(def d : dbcursor){ + if(d.shape.type=="Polygon"){ + def coordinates = d.shape.coordinates.get(0) + def points = [] + def index = 0 + for(def arrs : coordinates){ + if(index!=0 && index+1!= coordinates.size()){ + if(!points.contains(arrs)){ + points << arrs + }else{ + println d._id +" 重复数据:"+arrs + } + }else{ + points << arrs + } + index++ + } + if(points.size()!=coordinates.size()){ + d.shape.coordinates = [points] + def udata = new BasicDBObject() + udata.shape = d.shape + community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + } + }else{ + println d._id + def coordinateArrs = d.shape.coordinates + def multiPoints = [] + def updateFlag = false + for(def typeCoordinates : coordinateArrs){ + def pointArrs = [] + + for(def coordinates : typeCoordinates){ + def points = [] + def index = 0 + for(def arrs : coordinates){ + if(index!=0 && index+1!= coordinates.size()){ + if(!points.contains(arrs)){ + points << arrs + }else{ + println d._id +" 重复数据:"+arrs + updateFlag = true + } + }else{ + points << arrs + } + index++ + } + pointArrs << points + } + multiPoints << pointArrs + } + + if(updateFlag){ + d.shape.coordinates = multiPoints + def udata = new BasicDBObject() + udata.shape = d.shape + community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + } + } + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 9480fb72a..648d1df2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -93,12 +93,23 @@ class DevProcess { } def aiDevTown(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d -> - def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出") - def message = res.data.replaceAll("\n", "") - d['F'] = res.code - d['G'] = message - println d['E'] +" "+res.code + " "+message + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d -> +// def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出") +// def message = res.data.replaceAll("\n", "") +// d['F'] = res.code +// d['G'] = message +// println d['E'] +" "+res.code + " "+message + def q = new BasicDBObject("name", d['E']).append("type","乡镇街道") + def dbcursor = DEV_MAPPING_CONN.find(q) + if(dbcursor.count()>0){ + def list = [] + for(def db : dbcursor){ + list << db.township + } + + d['H']= list.join(",") + } + }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictLoopInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictLoopInfo.groovy new file mode 100644 index 000000000..be1a4b729 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictLoopInfo.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.looplineinfo.InPolygon +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.awt.geom.Point2D + +class DistrictLoopInfo { + + DBCollection LATLNG_POLYGON_COLL = MongoUtils.getCollectionByName("fangjia", "latlng_polygon", "mongo.db.fangjia.com") + + static main(args) { + def task = new DistrictLoopInfo() + task.run() + } + + def run(){ + def q = new BasicDBObject() + q.put("name", new BasicDBObject('$in', ["内环","中环","外环","郊环"])) + def loopShapes = LATLNG_POLYGON_COLL.find(q) + def loopMap = [:] + for(def loop : loopShapes){ + loopMap.put(loop.name, loop) + } + def districts = CommonDatautils.findAllDistricts("上海", "district", ["city", "region", "name", "lat", "lng", "loop_line_info"]) + println districts.size() + def index = 0 + for(def d : districts){ + index++ + if(d.lng){ + def lng = d.lng/10000000000.0 + def lat = d.lat/10000000000.0 + Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6, lng), DataHandleUtils.getDoubleRound(6, lat)); + def loop1 = loopMap.get("内环") + def flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates)) + def loop_line_info = "" + if(flag){ + loop_line_info = "内环内" + }else{ + loop1 = loopMap.get("中环") + flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates)) + if(flag){ + loop_line_info = "内中环" + }else{ + loop1 = loopMap.get("外环") + flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates)) + if(flag){ + loop_line_info = "中外环" + }else{ + loop1 = loopMap.get("郊环") + flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates)) + if(flag){ + loop_line_info = "外郊环" + }else{ + loop_line_info = "郊环外" + } + } + } + } + println index +" "+d.region +" "+d.name+" 原:"+d.loop_line_info+" 现:"+loop_line_info+" lng:"+lng+","+lat + def udata =new BasicDBObject() + udata.put("loop_line_info", loop_line_info) + DBUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set',udata)) + }else{ + println index + " "+d.region +" "+d.name+"无经纬度信息" + } + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy index 74d0ddbcb..63839d995 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictRepeatBatch.groovy @@ -22,7 +22,47 @@ class DistrictRepeatBatch { static main(args) { def task = new DistrictRepeatBatch() // task.run() - task.resultProcess() +// task.resultProcess() + task.processRepeatAndAlias() + } + + def processRepeatAndAlias(){ + ExcelUtils.modify("D:\\数据\\小区\\重复小区\\结果\\重复可疑小区-南京.xlsx",{d-> + if(d['H'] && d['H'] == "删除"){ + def obj = CommonDatautils.findDistrictById(d['A']) + if(!obj){ + obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + } + if(obj){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + saveOptLog(obj.city, obj.region, obj.name, "remove") + } + }else{ + if(d['I']){ + def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(obj){ + def arrs = d['I'].split(",") + if(arrs){ + def alias = obj.alias + if(!alias){ + alias = [] + } + alias.addAll(arrs) + alias = alias.unique() + + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + udata.put("alias", alias) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + saveOptLog(obj.city, obj.region, obj.name, "update") + } + } + } + } + },0) } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index 8a5fc6eec..1c45b42e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -17,7 +17,7 @@ import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection -class GXDdistrictMatch { +class 8GXDdistrictMatch { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 31105176e..19d13330b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -29,8 +29,8 @@ class TownMiddleProcess { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL) - DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL) - + DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping_0526", MONGO100_URL) + DBCollection TEMP_ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL) DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") @@ -65,7 +65,7 @@ class TownMiddleProcess { //修改乡镇街道别名 // updateTownAlias() //中间表同步到100 - synchTown() +// synchTown() //添加地级市标志 // addCityFlag() // addPinyin() @@ -101,11 +101,22 @@ class TownMiddleProcess { // updateNewRegionPoyline()//更新新的区域围栏 - updateMappingPolyline() +// updateMappingPolyline() + //批量修改多边型围栏数据结构 +// updatePolylineBatch() +// test2dshape() + + //将省会城市更新到数据库中 +// updateProvinceCenter() + //城市级别更新到库中 +// updateCityLevel() + //城市顺序 +// updateCityNum() } + def createProvince(){ def cat = "省" def tjjMaps = findAllProcincesFromTJJ(cat) @@ -457,16 +468,17 @@ class TownMiddleProcess { def datas = [] for(def d : dbcursor){ datas << d - if (datas.size() >= 2000) { + if (datas.size() >= 5000) { ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); // ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) +// ADMINISTRATIVE_MAPPING_COLL.insert(datas) println datas.size() datas.clear(); } } if (datas.size() >= 0) { - ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); + ADMINISTRATIVE_MAPPING_COLL.insert(datas) // ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) datas.clear(); } @@ -1449,4 +1461,123 @@ class TownMiddleProcess { } + def updatePolylineBatch(){ + def q = new BasicDBObject() + q.put("shape",new BasicDBObject('$exists', true)) + q.put("update_shape",new BasicDBObject('$exists', false)) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100) + for(def d : dbcursor){ + def type = d.shape.type + def udata = new BasicDBObject() + def shapeMap = [:] + if(type == "polygon"){ + shapeMap.put("type", "Polygon") + shapeMap.put("coordinates", d.shape.coordinates) + udata.put("shape", shapeMap) + udata.put("update_shape", 1) + ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata)) + }else if(type == "MultiPolygon"){ + def coordinates = d.shape.coordinates + def new_coors = [] + for(def coors : coordinates){ + new_coors << [coors] + } + shapeMap.put("type", "MultiPolygon") + shapeMap.put("coordinates", new_coors) + udata.put("shape", shapeMap) + udata.put("update_shape", 1) + ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata)) + } + } + } + + def test2dshape(){ + def q = new BasicDBObject() + q.put("street","花亭路街道") +// q.put("shape",new BasicDBObject('$exists', true)) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100) + for(def obj : dbcursor){ + try{ + TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj) + }catch(Exception e){ + def coordinates = obj.shape.coordinates + for(def coors : coordinates){ + for (int i = 0; i < coors.size(); i++) { // 改为索引遍历 + def coorsArr = coors[i] + def index = 0 + def new_coors = [] + for(def points : coorsArr){ + index ++ + if(!new_coors.contains(points)){ + new_coors << points + }else{ + if(index<=coorsArr.size()){ + new_coors << points + break; + } + } + } + if(new_coors.size()!=coorsArr.size()){ + coors[i] = new_coors + println "[修改] "+ obj.city +" "+obj.street +" 原:"+coorsArr.size()+" 现:"+new_coors.size() + } + } + } + try{ + TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj) + }catch(Exception ex){ + println "仍有问题:"+obj.city +" "+obj.street+" "+ex.getMessage() + } + + } + + } + } + + def updateProvinceCenter(){ + ExcelUtils.modify("D:\\数据\\数据建设\\城市\\省会城市.xlsx",{d-> + def city = d['C'] + if(city){ + def q = new BasicDBObject("type","城市").append("city_short", city) + def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + if(obj){ + def udata = new BasicDBObject("capital_city", "是") + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata)) + } + } + }) + } + + def updateCityLevel(){ + ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市等级.xlsx",{d-> + def city = d['A'] + if(city){ + def q = new BasicDBObject("type","城市").append("city_short", city) + def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + if(obj){ + def udata = new BasicDBObject("city_level", d['B']) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata)) + }else{ + println city +" 没找到" + } + } + }) + } + + def updateCityNum(){ + ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市排序.xlsx",{d-> + def city = d['A'] + if(city){ + def q = new BasicDBObject("type","城市").append("city_short", city) + def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + if(obj){ + def udata = new BasicDBObject("city_position", d['B'].toString().toDouble().toInteger()) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata)) + }else{ + println city +" 没找到" + } + } + }) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaodeCityRange.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaodeCityRange.groovy index 4e00ba591..7c5383cd7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaodeCityRange.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaodeCityRange.groovy @@ -18,16 +18,17 @@ class GaodeCityRange { } def run(){ - def citys = CommonDatautils.findAllCity() +// def citys = CommonDatautils.findAllCity() def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"] + def citys = ["新星","胡杨河","白杨"] HttpRequest request = new HttpRequest() def rows = [] citys.each{c-> if(!citys1.contains(c)){ c = c.replace("州","") - JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") + JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=66e6771767ab293ace441553c36621c1&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") if(resp.containsKey("districts")){ def datas = resp.get("districts") for(def d : datas){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy index 030658e63..a1671b461 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.location import com.fangjia.core.RegionData import com.fangjia.core.util.HttpRequest +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.Bytes @@ -22,6 +23,11 @@ class RegionPointCrawl { static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + static DBCollection TOWN_CENTERCOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_center") + + public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + + static main(args) { def task = new RegionPointCrawl(); //23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省 @@ -40,8 +46,12 @@ class RegionPointCrawl { // println "区域围栏 完成" // } - task.statSame() +// task.statSame() + //"福建省","河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","山东省","湖北省","湖南省","江西省","江苏省","安徽省" + def provinces = ["河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","青海省","山东省","福建省","浙江省","台湾省","河南省","湖北省","湖南省","江西省","江苏省","安徽省","广东省","海南省","四川省","贵州省","云南省","北京市","上海市","天津市","重庆市","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] + task.getTownshipCenter(provinces) + task.addTownCenter(provinces) } def crawlCityAndDistrictCode(def province){ @@ -210,7 +220,7 @@ class RegionPointCrawl { jo = JSONObject.fromObject(html) if(jo.infocode!="10000"){ errorCount ++ - Thread.sleep(30*1000) + Thread.sleep(3*1000) // println " 不可用Key: "+key }else{ errorCount = 100 @@ -228,15 +238,15 @@ class RegionPointCrawl { static String getRandomKey2() { def KeyList = [ - "2af0b8533415a612b98d59bb93d664b9", - "0246c57bbcf24aa246c207b3c99dfe41", - "042328b3f0fa5daf426802d379521d87", - "055d4f1b5b64a2bce18f4fe1f15c2639", - "d5484f5170502405d88bbaffe1b496f0", - "e74ccc4258079d2200199695110cceb1", - "f58629fc799fbc8537ca85046c6d44a5", - "291073f17ee0b963ccb1927ed92bf265", - "2a1fb75990f2b39f6221e50892dd3988" +// "2af0b8533415a612b98d59bb93d664b9", +// "0246c57bbcf24aa246c207b3c99dfe41", +// "042328b3f0fa5daf426802d379521d87", +// "055d4f1b5b64a2bce18f4fe1f15c2639", +// "d5484f5170502405d88bbaffe1b496f0", +// "e74ccc4258079d2200199695110cceb1", +// "f58629fc799fbc8537ca85046c6d44a5", + "eb69e4ad2a9cdd31f163a0ed32effe6e" +// "2af0b8533415a612b98d59bb93d664b9" ] def count = RandomUtils.nextInt(KeyList.size()) @@ -300,4 +310,117 @@ class RegionPointCrawl { } return map; } + + + def getTownshipCenter(provinces){ + for(def p : provinces){ + def query = new BasicDBObject("province", p) + def count = TOWN_CENTERCOLL.count(query) + if(count){ + println p +" 已存在" + }else{ + def ret = polygon(p, "3", "base"); + if(ret){ + def citys = ret[0].districts + if(citys){ + def allRets = [] + for(def c : citys){ + def cityTowns = getTownByCity(p, c.name,c.districts) + allRets.addAll(cityTowns) + } + saveTownCenter(allRets) + println p +" 保存:"+allRets.size() + } + }else{ + println p+ "无结果" + } + } + + } + } + + def getTownByCity(prov, city,distDatas){ + def rets = [] + for(def d : distDatas){ + def townDatas = d.districts + for(def town : townDatas){ + town.province = prov + town.district = d.name + town.city = city + rets << town + } + } + return rets + } + + + def saveTownCenter(allRets){ + for(def r : allRets){ + def obj = new BasicDBObject() + obj.put("province", r.province) + obj.put("city", r.city) + obj.put("district", r.district) + obj.put("town", r.name) + obj.put("center", r.center) + obj.put("level", r.level) + TOWN_CENTERCOLL.save(obj) + } + } + + def addTownCenter(provinces){ + for(def p : provinces){ + def q = new BasicDBObject("province",p).append("type","乡镇街道") +// q.put("location", new BasicDBObject('$exists', false)) + def dbcursor = administrative_mapping_coll.find(q) + for(def obj : dbcursor){ + + def tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", obj.street) + def dbo = TOWN_CENTERCOLL.findOne(tq) + if(!dbo){ + def new_street = obj.street.replace("管理处","") + tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", new_street) + dbo = TOWN_CENTERCOLL.findOne(tq) + } + if(dbo){ + def arrs = dbo.center.split(",") + def location = [:] + location["lon"] = arrs[0].toDouble() + location["lat"] = arrs[1].toDouble() + if(!obj.location){ + def udata = new BasicDBObject("location", location) + administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + println("【修改】"+obj.city +" "+ obj.district+" "+obj.street) + }else{ + def distance = com.fangjia.script.util.GeoUtils.calculateDistance(location.lat, location.lon, obj.location.lat, obj.location.lon); + if(distance>1000){ + def udata = new BasicDBObject("location", location) + println(obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 经纬度中心点相距1公里以外 原:"+obj.location.lon+","+obj.location.lat+" 新:"+dbo.center+" 距离:"+distance) + administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + } + }else{ + + def esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street) + if(!esobj){ + def short_prov = obj.province.replace("省", "") + if(obj.street.contains(short_prov)){ + esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street.replace(short_prov, "")) + } + } + if(esobj){ + + def location = [:] + location["lon"] = esobj.location.lon.toDouble() + location["lat"] = esobj.location.lat.toDouble() + def udata = new BasicDBObject("location", location) + administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + println("【修改】"+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street+" "+ esobj.cat+" "+esobj.name +" es poi ") + }else{ + println(obj.province+" "+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 没有找到") + } + + } + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/NonghangDataExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/NonghangDataExport.groovy new file mode 100644 index 000000000..fab3ed33b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/NonghangDataExport.groovy @@ -0,0 +1,15 @@ +package com.fangjia.script.gj.test + +class NonghangDataExport { + + static main(args) { + def task = new NonghangDataExport(); + task.run() + } + def run(){ + def citys = ["临沂","广州", "深圳"] + for(def c : citys){ + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/DistrictMonthlyCalc.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/DistrictMonthlyCalc.groovy index 37352f7ee..72dc0662d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/DistrictMonthlyCalc.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/DistrictMonthlyCalc.groovy @@ -21,15 +21,15 @@ public class DistrictMonthlyCalc{ public static void main(String[] args) { - List cities = ["上海"]; - def allCity = CommonUtils.findAllCity() - cities.addAll(allCity) + List cities = ["襄阳"]; +// def allCity = CommonUtils.findAllCity() +// cities.addAll(allCity) def districts = [] cities.each {def city-> districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("name", 1).append("region", 1)).batchSize(2000).toList() println(city) } - ExecutorService executorService = Executors.newFixedThreadPool(20) + ExecutorService executorService = Executors.newFixedThreadPool(1) districts.each {BasicDBObject district-> executorService.submit(new Calculator(district)) } @@ -43,7 +43,7 @@ public class DistrictMonthlyCalc{ class Calculator extends Thread{ static DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") - static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat_new", "mongo.db.fangjia.com") final BasicDBObject district; public Calculator(final BasicDBObject district){ @@ -52,14 +52,22 @@ class Calculator extends Thread{ @Override void run() { - REGION_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo-> + def q = new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name) + q.put("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01", "yyyy-MM-dd"))) + REGION_STAT.find(q).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo-> Date weekend = dbo.weekend if(DateUtil.isLastSundayOfMonth(weekend)){ def trend = new BasicDBObject("city", district.city) trend.append("region", district.region) trend.append("district_name", district.name) trend.append("date", weekend.format("yyyyMM").toInteger()) - REGION_STAT_TREND.update(trend , new BasicDBObject('$set', new BasicDBObject("avg_price", dbo.avg_price)),true,false) + def udata = new BasicDBObject() + if(dbo.avg_price_2b && dbo.value_type!=1){ + udata.put("avg_price", dbo.avg_price_2b) + }else{ + udata.put("avg_price", dbo.avg_price) + } + REGION_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false) } } REGION_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("data", 1)).each {BasicDBObject trend->