From 46211a20b97ea7f8a9242f19b107e5b9c895780e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 6 Sep 2023 13:26:07 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataForsaleBatch.groovy | 8 +- .../ronghe/ExportDistrictRongHe.groovy | 40 +++++ .../script/commData/CommonDatautils.groovy | 2 +- .../script/fuzhou/AmapAddressUpdate.groovy | 149 ++++++++++++++++-- .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 38 ++++- .../script/gj/city/CityStatExport.groovy | 24 ++- .../gj/xiamen/GaoDePoiDataProcess.groovy | 106 ++++++++++++- .../gj/xiamen/NetworkDistanceBatch.groovy | 42 ++++- .../script/gj/xiamen/SeaweedDataSynch.groovy | 26 ++- .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- .../fangjia/script/wenyan/temp/ZhuoXin.groovy | 8 +- 11 files changed, 410 insertions(+), 35 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 10822463f..b1b8447f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -85,6 +85,7 @@ class BigdataForsaleBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def startDate = lastMonth+"18" def endDate = curMonth+"19" println "房源时间:"+startDate+" "+endDate @@ -196,6 +197,7 @@ class BigdataForsaleBatch extends BaseBatch{ m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall); m.direction_code = CcbCodeUtils.getDirectionCode(m.direction) m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment); + m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure) m.address = rMap.address m.build_structure_code = struCode m.house_structure_code = null//getHouseStructureCode(); @@ -311,7 +313,7 @@ class BigdataForsaleBatch extends BaseBatch{ data << m.floor==null?"":m.floor data << m.height==null?"":m.height data << m.estate_type==null?"":m.estate_type - data << m.house_structure_code==null?"":m.house_structure_code + data << m.room_structure_code==null?"":m.room_structure_code data << m.build_structure_code==null?"":m.build_structure_code data << m.area==null?"":m.area data << "" @@ -670,6 +672,7 @@ class BigdataForsaleBatch extends BaseBatch{ rowMap.room = seawater.room=="null"?null:seawater.room rowMap.hall = seawater.hall=="null"?null:seawater.hall rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure if(CcbCommonUtils.rootMap.get(seawater.root)){ rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) }else{ @@ -731,7 +734,7 @@ class BigdataForsaleBatch extends BaseBatch{ QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) - .must(date_range).must(area_range).must(date_range_mdate) + .must(date_range).must(area_range).must(date_range_mdate).must(QueryBuilders.termQuery("root", "lianjia.com")) if(!alias){ alias = [] @@ -882,6 +885,7 @@ class BigdataForsaleBatch extends BaseBatch{ rowMap.room = seawater.room=="null"?null:seawater.room rowMap.hall = seawater.hall=="null"?null:seawater.hall rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure if(CcbCommonUtils.rootMap.get(seawater.root)){ rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) }else{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index c8d486778..4ab006255 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -52,6 +52,8 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataBuildingFJW()// exportBigdataCommfeeGj()// + //exportGuidedPrice()//10月10号再同步数据 + exportBigdataCommbuildstruGJ()// exportBigdataCommhousetypeGJ()// exportBigdataCommbuildyearGj()// @@ -188,6 +190,44 @@ class ExportDistrictRongHe extends BaseBatch{ CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" } + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_dt + data << r.gujia_bds_drc_prc + data << r.gujia_add_psn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + + println "GUIDED_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "GUIDED_PRICE_GJ 上传Ftp完成......" + } + + def exportBigdataCommbuildstruGJ(){ def allRows = [] for(def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index c59cdcd87..eee6db626 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -255,7 +255,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","华润佘山九里二期") +// q.put("name","文华楼") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy index 0be2c85d2..5a5c5f63f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.fuzhou +import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils @@ -38,10 +39,9 @@ class AmapAddressUpdate { static main(args) { def task = new AmapAddressUpdate() - def path = "d:\\数据\\小区\\福建省小区地址与高德地图V2.xlsx" -// def addr = task.formatAddress("福州", "北三环路201号") -// println addr - task.run(path) + def path = "d:\\数据\\小区\\重庆小区地址与高德地图V1.xlsx" + +// task.run(path) // task.similarMatch(path) // task.addressRun(path) // task.addressSource(path) @@ -50,11 +50,15 @@ class AmapAddressUpdate { // task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较 // task.updateData(path) // task.updateLocation(path)//更新经纬度 + + task.matchNoSameRegionDist() } def run(path){ - def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"]//CommonDatautils.findAllCity() + //"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田" + def citys = ["重庆"]//CommonDatautils.findAllCity() def allrets = [] - allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"] + allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度","得分","匹配类型","小区经纬度与高德距离","操作","来源","地址与小区经纬度距离","备注","经纬度是否修改","高德类型"] + for (def city : citys) { def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"]) def rets = findAmapDetail(disticts) @@ -63,6 +67,17 @@ class AmapAddressUpdate { ExcelUtils.write(path, allrets, "数据") } + def matchNoSameRegionDist(){ + def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"] + for (def city : citys) { + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address","estate_type2", "addr_alias","lat", "lng"]) + def rets = findDiffRegionAmapDetail(disticts) + ExcelUtils.write("D:\\数据\\小区\\区域有问题小区.xlsx", rets, city) + } + + } + + def amapRun(){ def city = "福州市" def datas = findAllAmapByCity(city) @@ -177,7 +192,15 @@ class AmapAddressUpdate { row << formatAddress(d.city, dbo.address) row << dbo.lng row << dbo.lat + row << 1 row << dbo.match_type + if(d.lng && dbo.lng){ + def distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble()); + row << distance + }else{ + row << "" + } + }else{ row << "" row << "" @@ -186,6 +209,8 @@ class AmapAddressUpdate { row << "" row << "" row << "" + row << "" + row << "" } println totalCount.get() +' '+row rets << row @@ -201,6 +226,75 @@ class AmapAddressUpdate { return rets } + + + + def findDiffRegionAmapDetail(districts){ + def rets = Collections.synchronizedList(new ArrayList()); + rets << ["城市","区域","名称","地址","物业类型","地图"] + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + for(def i=0;i + def index = 0 + ExcelUtils.modify(path, { row -> if(row['R'] && row['R']=="是"){ + index++ def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") if(dist1){ def udata = new BasicDBObject() @@ -514,8 +642,9 @@ class AmapAddressUpdate { row['J'] = dbo.lng row['K'] = dbo.lat row['L'] = dbo.score - row['M'] = dbo.distance - row['S'] = dbo.type + row['N'] = dbo.distance + row['M'] = dbo.match_type + row['T'] = dbo.type } } } @@ -539,7 +668,7 @@ class AmapAddressUpdate { def mapName = rs.getString("name").replace("·","") println name +" "+ mapName - if(rs.getString("adname").contains(region)){ + if(rs.getString("adname") && rs.getString("adname").contains(region)){ map.name = rs.getString("name") map.address = rs.getString("address") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index e343714fd..842e0afe1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -230,12 +230,46 @@ class CcbCodeUtils { } return "11" } - //室内结构 - //平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23 + static def getHouseStructureCode(){ return null } + //室内结构 + //平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23 + static def getRoomStructureCode(room_structure){ + if(room_structure){ + if(room_structure.contains("平层")){ + return "11" + }else if(room_structure.contains("错层")){ + return "12" + }else if(room_structure.contains("复式")){ + return "13" + }else if(room_structure.contains("阁楼")){ + return "14" + }else if(room_structure.contains("团结户")){ + return "15" + }else if(room_structure.contains("老式房")){ + return "16" + }else if(room_structure.contains("地下室")){ + return "17" + }else if(room_structure.contains("带阁楼")){ + return "21" + }else if(room_structure.contains("带车库")){ + return "22" + }else if(room_structure.contains("带车位")){ + return "23" + }else if(room_structure.contains("带地下室")){ + return "24" + }else if(room_structure.contains("带储藏室")){ + return "25" + }else if(room_structure.contains("跃层")){ + return "18" + } + } + return null; + } + //租赁方式 整租:0 合租:1 转租:2 static def getRentTypeCode(def rentType){ if(rentType){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy index 4781397c2..3289104fe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/city/CityStatExport.groovy @@ -35,15 +35,23 @@ class CityStatExport { } def addFieldStat(){ - ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d -> - def cityInfoDto = CommonDatautils.findCityInfo(d['B']) - if(cityInfoDto){ - def detailMap = cityInfoDto.district_detail - d['D'] = detailMap.address.toDouble()*100 - d['E'] = 100 - d['F'] = cityInfoDto.price_coverage.toDouble()*100 + def fileName = "D:\\数据\\fjw.xlsx" + ExcelUtils.multiThreadingModify(fileName, { d -> + if(!d['B']){ + def city = RegionData.formatCity(d['A']) + if(city){ + def cityInfoDto = CommonDatautils.findCityInfo(city) + if(cityInfoDto){ + d['B'] = cityInfoDto.existing_district_count + def detailMap = cityInfoDto.district_detail + // d['D'] = detailMap.address.toDouble()*100 + // d['E'] = 100 + d['D'] = cityInfoDto.price_coverage.toDouble()*100 + } + } } - }) + + },1) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 3ec5214f7..55100de2c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -6,12 +6,15 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl import org.bson.types.ObjectId import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse @@ -57,7 +60,10 @@ class GaoDePoiDataProcess extends BaseBatch{ @Override void process(Object args) { // synchMapPoi(); - synchCompany() +// synchCompany() +// poiCatProcess() +// addressAndCatProcess() + synchNetwork() } def synchCompany(){ @@ -368,5 +374,103 @@ class GaoDePoiDataProcess extends BaseBatch{ return townMap } + def poiCatProcess(){ + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\数据.xlsx",{d-> + def cat = d['B'] + if(cat && cat.size()>50){ + println cat + def arrs = cat.split(";") + def newCat = "" + if(arrs.size()>3){ + newCat = arrs[0]+";"+arrs[1]+";"+arrs[2] + def udata = new BasicDBObject() + udata.put("cat",newCat) + udata.put("type",arrs[0]) + udata.put("secondary_type",arrs[1]) + udata.put("small_type",arrs[2]) + MAPPOI_CONN.update(new BasicDBObject("cat", cat), new BasicDBObject('$set', udata), false, true) + } + } + }) + } + + + def addressAndCatProcess(){ + def datas = FileUtils.readTxt("C:\\Users\\gaojuan\\Downloads\\BIGDATA_POI_v1.bad") + datas.each{d-> + def arrs = d.split("@\\|@") + if(arrs){ + def udata = new BasicDBObject() +// def address = arrs[5] +// if(address.size()>=120){ +// udata.put("address", "") +// } + def cat = arrs[8] + def catArrs = cat.split(";") + if(catArrs.size()>3){ + cat = catArrs[0]+";"+catArrs[1]+";"+catArrs[2] + if(cat.size()>=45){ + println "取前3依然超:"+cat + }else{ + udata.put("type",catArrs[0]) + udata.put("secondary_type",catArrs[1]) + udata.put("small_type",catArrs[2]) + udata.put("cat", cat) + } + + }else{ + println cat + } + + if(udata){ + MAPPOI_CONN.update(new BasicDBObject("_id", arrs[0]), new BasicDBObject('$set', udata), false, true) + } + } + } + } + + def synchNetwork(){ + def datas = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx") + def rets = [] + for(def d : datas){ + def location = d['I'] + def locMap = [:] + if(location){ + def arrs = location.split(",") + locMap.lon = arrs[0].toString().toDouble() + locMap.lat = arrs[1].toString().toDouble() + } + rets << ["id":new ObjectId().toString(), "name":d['B'], "address":d['C'], "location":locMap] + } + + saveEs("xm_network", rets) + } + + def saveEs(index,rows){ + BulkRequest request = new BulkRequest(); + int batchSize = 1000; + int processedCount = 0; + def client = ElasticUtils.getClient("192.168.0.12") + for (int i = 0; i < rows.size(); i++) { + def map = rows[i] + request.add(new IndexRequest(index, "_doc", map.id).source(map, XContentType.JSON)); + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + println "结束时间 "+new Date() + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/NetworkDistanceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/NetworkDistanceBatch.groovy index bd8531aae..bf848d990 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/NetworkDistanceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/NetworkDistanceBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.xiamen +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeoUtils import com.fangjia.datastore.driver.DSCursor import com.fangjia.script.BaseBatch @@ -9,6 +10,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.rabbitmq.client.impl.AMQImpl import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -27,6 +29,8 @@ class NetworkDistanceBatch extends BaseBatch{ final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43") + final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "192.168.0.43") + static main(args) { runBatch(new NetworkDistanceBatch(), args) } @@ -34,11 +38,45 @@ class NetworkDistanceBatch extends BaseBatch{ @Override void process(Object args) { - def networks = ExcelUtils.read("D:\\数据\\厦门\\网点.xlsx", 0, 1) + def networks = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx", 0, 1) def citys = ["厦门","三明","宁德","泉州","福州","漳州","南平","龙岩","莆田","重庆"] for(def c : citys){ run(c, "district", networks) } +// saveMapData() + } + + def saveMapData(){ + ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\厦门银行鹭通支行.xlsx",{d-> + if(d['C']){ + def arrs = d['C'].split(",") + def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + def obj = new BasicDBObject() + obj.put("network_name", d['B']) + obj.put("ntk_gd_lng", arrs[0].toString().toDouble()) + obj.put("ntk_gd_lat", arrs[1].toString().toDouble()) + obj.put("ntk_bd_lng", lng2) + obj.put("ntk_bd_lat", lat2) + obj.put("comm_id", d['D']) + obj.put("city", d['E']) + obj.put("region", d['F']) + obj.put("name", d['G']) + obj.put("distance", d['I'].toString().toDouble().toInteger()) + obj.put("time", d['K'].toString().toDouble().toInteger()) + obj.put("gh_distance", d['J'].toString().toDouble().toInteger()) + def dist_loc_arr = d['H'].split(",") + + def baidu_dist = BaiDuApiUtils.geoconv(dist_loc_arr[0].toString().toDouble(), dist_loc_arr[1].toString().toDouble(), 3) + obj.put("dist_gd_lng", dist_loc_arr[0].toString().toDouble()) + obj.put("dist_gd_lat", dist_loc_arr[1].toString().toDouble()) + obj.put("dist_bd_lng", baidu_dist.x) + obj.put("dist_bd_lat", baidu_dist.y) + + MAP_NETWORK_COLL.save(obj) + } + }) } def run(city, cat, networks){ @@ -81,7 +119,7 @@ class NetworkDistanceBatch extends BaseBatch{ def lastRest = null def minDistance = Integer.MAX_VALUE for(def row : networks){ - def locationstr = row['D'] + def locationstr = row['E'] if(locationstr){ def arrs = locationstr.split(",") def distance = GeoUtils.calculateDistance(dist.location.lon, dist.location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 798a752e3..2046c0778 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -23,16 +23,19 @@ class SeaweedDataSynch { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) - + public static def provinceMap = CommonDatautils.findAllCityProvince() static main(args) { def task = new SeaweedDataSynch() +// def addr = ["a","b",'c',"d","e","f"] +// println addr[0..4] task.run() // println new ObjectId("622c433afb459ab08fdab460").getDate() } def run(){ + def citys = CommonDatautils.findAllCity() def index = 0 for(def c : citys){ @@ -45,7 +48,7 @@ class SeaweedDataSynch { } def updating(city, cat){ - def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"]) + def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"]) def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) @@ -92,13 +95,28 @@ class SeaweedDataSynch { def createData(dist){ def map = [:] map._id = dist._id + map.province = provinceMap.get(dist.city) map.city = dist.city map.region = dist.region map.name = dist.name map.address = dist.address map.block = dist.block - map.alias= dist.alias - map.addr_alias = dist.addr_alias + if(dist.mapping_alias){ + if(dist.mapping_alias.size()>5){ + map.alias= dist.mapping_alias[0..4] + }else{ + map.alias= dist.mapping_alias + } + } + + if(dist.addr_alias){ + if(dist.addr_alias.size()>5){ + map.addr_alias = dist.addr_alias[0..4] + }else{ + map.addr_alias = dist.addr_alias + } + } + map.category = dist.cat map.estate_type = dist.estate_type2 map.status = dist.status diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 1e51359de..abf289d1f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 10 + def nThread = Runtime.getRuntime().availableProcessors() * 2 ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) final double total = body.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy index 8817fefd3..2e1012315 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ZhuoXin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20230701", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230801", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230801", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230901", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -23,7 +23,7 @@ while (startDate.before(endDate)) { def nextDate = calendar.getTime() count++ ///property/addressEvaluate /address/financialRating /v2/property/detail - OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/address/financialRating"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) // if (!hashSet.contains(result)) { def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] @@ -39,6 +39,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情") +ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202309.xlsx", data, "财力评级")