diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 249590993..13fcabfc3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -33,7 +33,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")); + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); @@ -61,56 +61,58 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = CcbCommonUtils.getRunningCitys(args) +// def citys = CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } println "curDate:"+curDate -// def allCitys = CommonDatautils.findAllCity() -// for(def city : allCitys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } + def allCitys = ["福州"] //CommonDatautils.findAllCity() + for(def city : allCitys){ +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } + } + + +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 +// for(def city : citys){ +// index ++ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// // } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def index = 0 - for(def city : citys){ - index ++ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } - - } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成 " + rows.size() - - saveToPgSql(rows) +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成 " + rows.size() +// +// saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -649,16 +651,21 @@ class BigdataDealBatch extends BaseBatch{ def final AtomicInteger count_ = new AtomicInteger(0) for(def i=0;i + def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306] + def deal_pes = [] + def third_pes = [] + for(def d : dates){ + def map = findPeGteCity(row['B'], d) + if(map.deal_pe){ + deal_pes << map.deal_pe + } + if(map.third_pe){ + third_pes << map.third_pe + } + } + if(deal_pes){ + row['F'] = deal_pes.sum()/deal_pes.size() + } + if(third_pes){ + row['G'] = third_pes.sum()/third_pes.size() + } + println row['B'] + " "+row['F']+" "+row['G'] + }) + + } + + + + def findPeGteCity(cityName, yearmonth){ + def sql = "SELECT gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}" + def map = [:] + pgsqlInstance.eachRow(sql,{r-> + map.deal_pe = r.gujiamdltst_ppe15_pct + map.third_pe = r.gujiatrdpttstppe15pct + }); + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 591eb95bd..80f739637 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) // // //更新静态数据表 - updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") +// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy index f2a564cb7..2fd7e0c2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -17,4 +17,14 @@ class PgsqlCommUtils { }); return set } + + + public static def findAllDistrictCount(def cityid){ + def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def count = 0 + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 2ce980c49..0fd8eaa49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,8 +45,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -//def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", -// "postgres", "postgres", "org.postgresql.Driver") +def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") //def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"] //def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -1289,28 +1289,28 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx" -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['A']){ - def region = RegionData.formatRegion(d['B'], d['C']) - if(region){ - def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") - if(dist){ - d['G'] = dist.region - d['H'] = dist._id.toString() - } - } -// if(!d['F']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -//// +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['A']){ +// def region = RegionData.formatRegion(d['B'], d['C']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") // if(dist){ -//// d['H'] = dist.region -//// d['G'] = dist.name -// d['F'] = dist.status +// d['G'] = dist.region +// d['H'] = dist._id.toString() // } // } - - } -},0) +//// if(!d['F']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +////// +//// if(dist){ +////// d['H'] = dist.region +////// d['G'] = dist.name +//// d['F'] = dist.status +//// } +//// } +// +// } +//},0) @@ -1364,6 +1364,53 @@ def findGuidedPrice(sqlInstance, id){ return count } +def findPgDealPrice(c, pgsqlInstance, startDate){ + def sql = "select count(a.gujia_comm_id) from (SELECT distinct gujia_comm_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${c}' and txn_dt>='${startDate}') a" + def count = 0 + pgsqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + +citys = CcbCommonUtils.citys_all +def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"] +def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306] +def rows = [] +def headers = [] +headers << "城市" +headers.addAll(dates) +rows << headers +for(def c : citys){ + def row = [] + row << c + def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c)) +// for(def d : dates){ + def startDate = "2022-07-01" + def count = findPgDealPrice(c, pgsqlInstance, startDate) + row << count*100/disCount + +// def map = findSampleRate(pgsqlInstance, c, d) +// row << map.third +// } + rows << row +} +ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方") + + + + + +def findSampleRate(pgsqlInstance, cityName, yearmonth){ + def sql = "SELECT urbn_nm,gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}" + def map = [:] + pgsqlInstance.eachRow(sql,{r-> + map.deal = r.gujiamdltst_ppe15_pct + map.third = r.gujiatrdpttstppe15pct + }); + return map +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 9ed8300cc..5a38c237e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); -// cityRun(citys, startDate, endDate, cityCodeMap) -// regionRun(citys, startDate, endDate, cityCodeMap) + cityRun(citys, startDate, endDate, cityCodeMap) + regionRun(citys, startDate, endDate, cityCodeMap) districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy index cdad0ddb6..25560ff03 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -48,7 +48,7 @@ class HailanTest { } def run(){ - def inputfile = "D:\\数据\\微元素\\排查表.xlsx" + def inputfile = "D:\\数据\\微元素\\排查表-价格对比-新.xlsx" // addThirdPrice(inputfile) // match(inputfile) // tradeAvgPrice(inputfile) @@ -58,7 +58,8 @@ class HailanTest { // addHouse(inputfile) // stat(inputfile) // hailanStat(inputfile) - addFzFjwPrice(inputfile) +// addFzFjwPrice(inputfile) + addFjwTobPrice(inputfile) } def addHouse(inputfile){ @@ -748,6 +749,18 @@ class HailanTest { }) } + + def addFjwTobPrice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(d['F']){ + def dbo = CommonDatautils.queryDistrictPrice("福州", d['F'], d['G'], DateUtils.parseFromTightByPattern("2023-07-02","yyyy-MM-dd")) + if(dbo){ + d['Q'] = dbo.avg_price_2b + } + } + }) + } + def addFzFjwPrice(inputfile){ ExcelUtils.modify(inputfile, { row -> if(row['B']) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e8cc8e9f8..9bcbde0b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -252,7 +252,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "长滩美墅三期") +// q.put("name", "燃灯小区") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy index 7fe3204dd..5edea094e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy @@ -250,8 +250,23 @@ class DataHandleUtils { return sum; } - - - + + + public static isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街道|大道|巷|胡同|街|坊]\\d+(-\\d+)?(号|弄)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } + + + + + static main(args) { + def a = isStandardRoad("杨桥西路236号") + println a + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy index b604c38ff..530d8b36d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy @@ -5,7 +5,7 @@ import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoUtils import com.mongodb.BasicDBObject - +//删除别名是另一个小区的主名 class MasterNameInAlias { static main(args) { @@ -14,7 +14,7 @@ class MasterNameInAlias { } def run(){ - def citys = CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity().reverse() def allList = [] for(def city : citys){ def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy new file mode 100644 index 000000000..cecc0536d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy @@ -0,0 +1,616 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.sql.ResultSetMetaData +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AmapAddressUpdate { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static main(args) { + def task = new AmapAddressUpdate() + def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx" + def addr = task.formatAddress("福州", "北三环路201号") + println addr +// task.run(path) +// task.similarMatch(path) +// task.addressRun(path) +// task.addressSource(path) +// task.amapRun() + +// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较 +// task.updateData(path) +// task.updateLocation(path)//更新经纬度 + } + def run(path){ + def citys = ["福州"]//CommonDatautils.findAllCity() + def allrets = [] + allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"] + for (def city : citys) { + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"]) + def rets = findAmapDetail(disticts) + allrets.addAll(rets) + } + ExcelUtils.write(path, allrets, "数据") + } + + def amapRun(){ + def city = "福州市" + def datas = findAllAmapByCity(city) +// datas = datas[0..10] + println "Es总量:"+datas.size() + def rets = match(datas) + + ExcelUtils.write("d:\\数据\\小区\\福州高德数据匹配.xlsx", rets, "数据") + } + + def match(datas){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + rets << ["ID","高德城市","高德区域","高德名称","高德地址","匹配城市","匹配区域","匹配名称","匹配类型","匹配ID"] + for(def i=0;i +// println data.adname+" "+data.name +" "+data.address +" "+data.type +// if(name == data.name && data.adname.contains(region) && data.type.contains("type")){ +// data.formatAddress = AddressUtil.formatAddress(city, data.address) +// loc = data +// } +// }) + return map; + } + + + + + + def addressRun(inputfile){ + + ExcelUtils.multiThreadingModify(inputfile, { row -> +// if(row['H'] && row['H'].contains("口步行") && row['H'].contains("(")){ +// row['I'] = row['H'].substring(0, row['H'].indexOf("(")) +// } + if(row['J']){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def distance = null + if(dist1){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, row['J'].toString().toDouble(), row['K'].toString().toDouble()); + } + row['M'] = distance + } + + def gaodeRoadFlag = false + if(row['I']){ + gaodeRoadFlag = DataHandleUtils.isStandardRoad(row['I']) + } + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(!row['N'] && row['D'] && row['I']){ + if(row['D']==row['I']){ + row['N'] = 100 + }else if(row['E'] && gaodeRoadFlag && row['E'].contains(row['I'])){ + row['N'] = 1 + } + } + + if(!row['N'] && !dbRoadFlag && gaodeRoadFlag){ + row['N'] = 2 + } + + if(!row['N'] && dbRoadFlag && gaodeRoadFlag && row['I'].contains(row['D'])){ + row['N'] = 3 + } + + if(!row['N'] && dbRoadFlag && gaodeRoadFlag){ + row['N'] = 4 + } + + }) + } + + def addressSource(inputfile) { + + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(!row['N'] && row['D'] && !row['O']){ + def addresses = findAllEsData(row['A'], row['B'], row['C']) + if(addresses){ + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(dbRoadFlag){ + def list = [] + list << row['D'] + if(row['E']){ + def arr = row['E'].split(" ") + list.addAll(arr as List) + } + for(def addr : list){ + if(addresses.contains(addr)){ + row['O'] = "诸葛找房" + break + } + } + } + } + } + }) + } + + + def findAllEsData(city,region, name){ + def rets = [] + def sql = "select * from community where city='${city}' and name='${name}' and source='诸葛找房APP'" + rets = findDatas(region, sql) + if(!rets){ + rets = findAliasDatas(city,region, name) + } + return rets; + } + + def findDatas(region, sql){ + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + if(d.region && d.region.contains(region) && d.address){ + rets << d.address + } + }) + return rets; + } + + + def findAliasDatas(city, region, name){ + def must = [] + def term1 = ["match":["city":city]] + must << term1 + + def should = [] + should << ["match":["alias":name]] + def params = ["query":["bool":["must":must, "should":should]]] + def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/community/_doc/_search"); + def rets = []; + if(s) { + def doc = new groovy.json.JsonSlurper().parseText(s) + def hits = doc.hits + if(hits){ + hits = hits.hits + for(def h : hits){ + def d = h._source + if(d.region && d.region.contains(region) && d.address && d.source=="诸葛找房APP"){ + rets << d.address + } + } + } + } + return rets; + } + + def addrAndLocationDistance(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(!row['N'] && row['D']){ + println row['D'] + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(dbRoadFlag){ + def gloJson = GaoDeApiUtils.geo(row['A'], row['D']) + if(gloJson) { + def glocation = null + if (gloJson && gloJson.get("geocodes")) { + def geocodes = gloJson.get("geocodes") + def map = geocodes.get(0) + if(map.formatted_address.contains(row['D'])){ + def arrs = map.location.split(",") + + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def distance = null + if(dist1){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, arrs[0].toString().toDouble(), arrs[1].toString().toDouble()); + } + row['P'] = distance + } + } + } + } + } + }) + } + + def updateData(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['Q'] && row['Q'].contains("修改地址")){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def udata = new BasicDBObject() + if(row['Q'].contains("可替换原地址")){ + udata.put("address", row['I']) + }else if(row['Q'].contains("可新增地址别名")){ + udata.put("address", row['I']) + def addrAlias = dist1.addr_alias + if(!addrAlias){ + addrAlias = [] + } + addrAlias << dist1.address + udata.put("addr_alias", addrAlias) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata)) + } + + + + }) + } + + def updateLocation(path){ + ExcelUtils.multiThreadingModify(path, { row -> + + if(row['R'] && row['R']=="是"){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + if(dist1){ + def udata = new BasicDBObject() + def lat = null + def lng = null + if(row['J']){ + lng = row['J'].toString().toDouble() + lat = row['K'].toString().toDouble() + }else{ + def gloJson = GaoDeApiUtils.geo(row['A'], row['D']) + if(gloJson) { + if (gloJson && gloJson.get("geocodes")) { + def geocodes = gloJson.get("geocodes") + def map = geocodes.get(0) + if (map.formatted_address.contains(row['D'])) { + def arrs = map.location.split(",") + lng = arrs[0].toString().toDouble() + lat = arrs[1].toString().toDouble() + } + } + } + } + + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + def lng2 = baidu.x + def lat2 = baidu.y + Long lng_new = lng *10000000000L + Long lat_new = lat *10000000000L + udata.put("lng", lng_new) + udata.put("lat", lat_new) + if(lng2){ + Long lng2_new = lng2 *10000000000L + Long lat2_new = lat2 *10000000000L + udata.put("lng2", lng2_new) + udata.put("lat2", lat2_new) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata)) + } + + } + + }) + } + + def similarMatch(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def d = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + if(d && d.lat){ + def dbo = getSimilarAmapLocation(d.city+"市", d.region, d.name, d.address, d.lat/10000000000.0, d.lng/10000000000.0) + if(dbo){ + row['F'] = dbo.region + row['G'] = dbo.name + row['H'] = dbo.address + row['I'] = formatAddress(d.city, dbo.address) + row['J'] = dbo.lng + row['K'] = dbo.lat + row['L'] = dbo.score + row['M'] = dbo.distance + row['S'] = dbo.type + } + } + } + + }) + } + + + + def getSimilarAmapLocation(city, region, name, address, lat, lng){ + + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "'") + sqlBuffer.append(" and MATCH(name,'${name}')") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + def maxSocreObj = null + while (rs.next()) { + def map = [:] + def mapName = rs.getString("name").replace("·","") + + println name +" "+ mapName + if(rs.getString("adname").contains(region)){ + map.name = rs.getString("name") + map.address = rs.getString("address") + + map.region = rs.getString("adname") + map.type = rs.getString("type") + map.match_type = "名称相似" + def location = rs.getObject("location") + if(location){ + map.lng = location.x + map.lat = location.y + } + + double nameScore = 0 + double addScore = 0 + double loctionScore = 0 + def distance = com.fangjia.core.util.GeoUtils.calculateDistance(lng, lat, map.lng.toString().toDouble(), map.lat.toString().toDouble()); + map.distance = distance + if(map.name && name){ + nameScore = CosineSimilarAlgorithm.getSimilarity(map.name, name); + } + if(map.address && address){ + addScore = CosineSimilarAlgorithm.getSimilarity(map.address, address); + } + + if(hasColumn(rs, "shape")){ + def shape = rs.getObject("shape") + if(shape){ + def coordinates = shape.getObject("coordinates") +// def pointList = InsidePolygon.createPoint2(points); + def flag = InsidePolygon.isInArea(lng, lat, coordinates) + if(flag){ + loctionScore = 0.5 + } + } + }else{ + + if(distance < 500 && (nameScore>0.5 || addScore>0.5)){ + loctionScore = 0.3 + } + } + + def score = nameScore*0.3 + addScore*0.2 + loctionScore + map.score = score + if(!maxSocreObj){ + if(score>0){ + map.formatAddress = AddressUtil.formatAddress(city, map.address) + maxSocreObj = map + } + }else{ + println score +" "+maxSocreObj.score + if(maxSocreObj && score>maxSocreObj.score){ + map.formatAddress = AddressUtil.formatAddress(city, map.address) + maxSocreObj = map + } + } + } + } + return maxSocreObj; + } + + + def hasColumn(ResultSet rs, String columnName) { + ResultSetMetaData rsmd = rs.getMetaData(); + int columns = rsmd.getColumnCount(); + for (int x = 1; x <= columns; x++) { + if (columnName.equals(rsmd.getColumnName(x))) { + return true; + } + } + return false; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index dc389ffe5..585d07a4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch { } def calcMonthlyPrice(){ - def citys = CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 9cea6cf74..b6eab8c46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -2,6 +2,9 @@ package com.fangjia.script.fuzhou import com.fangjia.core.RegionData import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.JsonUtils +import com.fangjia.core.util.Message +import com.fangjia.core.util.MessageInfo import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils import com.fangjia.script.util.DateUtils @@ -9,26 +12,71 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONObject import org.apache.commons.io.FileUtils import org.bson.types.ObjectId DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") +import com.fangjia.core.util.HttpRequest -ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> - def city = RegionData.formatCity("余姚") - d['D'] = city - if(city){ - d['E'] = RegionData.formatRegion(city, d['C']) - } -// if(!d['D']){ -// def citys = CommonDatautils.getCityByRegion(d['A']) -// if(citys && citys.size()==1){ -// d['F'] = citys.get(0) -// } +//Message message = new Message(); +//message.setMsgtype("text"); +//message.setText(new MessageInfo("测试")); +//URL url = new URL("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"); +//// 建立http连接 +//HttpURLConnection conn = (HttpURLConnection) url.openConnection(); +//conn.setDoOutput(true); +//conn.setDoInput(true); +//conn.setUseCaches(false); +//conn.setRequestMethod("POST"); +//conn.setRequestProperty("Charset", "UTF-8"); +//conn.setRequestProperty("Content-Type", "application/Json; charset=UTF-8"); +//conn.connect(); +//OutputStream out = conn.getOutputStream(); +//String textMessage = JsonUtils.toJson(message); +//byte[] data = textMessage.getBytes("UTF-8"); +//out.write(data); +//out.flush(); +//out.close(); +//System.out.println(conn.getResponseCode()); +//InputStream ins = conn.getInputStream(); +//byte[] data1 = new byte[ins.available()]; +//ins.read(data1); +//System.out.println(new String(data1)); + +//HttpRequest request = new HttpRequest() +//def params = [:] +//params.put("","") +//request.post("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430","", "utf-8") + + +//sendMarkdownMessage("租金", "测试") +//static void sendMarkdownMessage(String title, String msg){ +// HttpRequest request = new HttpRequest() +// JSONObject requestBody = new JSONObject() +// JSONObject markdown = new JSONObject() +// markdown.put("title", title) +// markdown.put("text", msg) +// requestBody.put("markdown", markdown) +// requestBody.put("msgtype", "markdown") +// request.postWithBody("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430", requestBody, null, "UTF-8") +//} +//ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> +// def city = RegionData.formatCity("余姚") +// d['D'] = city +// if(city){ +// d['E'] = RegionData.formatRegion(city, d['C']) // } -}) +// +//// if(!d['D']){ +//// def citys = CommonDatautils.getCityByRegion(d['A']) +//// if(citys && citys.size()==1){ +//// d['F'] = citys.get(0) +//// } +//// } +//}) //def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1) //def m = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index db058c5a7..a3236b53a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -1,9 +1,14 @@ package com.fangjia.script.gj.district import com.fangjia.core.util.HttpRequest +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.HttpRequestUtil +import com.mongodb.DBObject import net.sf.json.JSONArray import net.sf.json.JSONObject +import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang.StringUtils class FangjiaClientUtils { @@ -153,4 +158,18 @@ class FangjiaClientUtils { return columnValMap; } + + public static def saveOrUpdateDistrict(data) { + String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data) +// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr) +// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr) + + com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr) + if (json.status == 0) { + return "成功" + }else if(json.status == 1){ + return json.msg + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy index 7107037b9..597926900 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy @@ -10,6 +10,9 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +/** + *处理地址在地址别名中有重复 + */ class SameDistName { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") @@ -18,13 +21,13 @@ class SameDistName { def task = new SameDistName() // task.run() //地址和地址别名完全一致 -// task.addressPocess() + task.addressPocess() } def addressPocess(){ - def citys = CommonDatautils.findAllCity(); + def citys = ["福州"]//CommonDatautils.findAllCity(); for(def c : citys){ def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy new file mode 100644 index 000000000..c92e387fe --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy @@ -0,0 +1,121 @@ +package com.fangjia.script.gj.location + + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 判断小区是否在城市范围内 + */ +class DistrictLocationBatch { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static DBCollection RANGECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline") + + static main(args) { + def task = new DistrictLocationBatch() +// task.run() + task.deleteDist() + } + + def deleteDist(){ + ExcelUtils.multiThreadingModify("d:\\数据\\小区\\非本城市小区.xlsx",{d-> + DBObject q = new BasicDBObject() + q.put("city", d['A']) + q.put("region", d['B']) + q.put("name", d['C']) + q.put("cat", 'district') + q.put("status", 0) + + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }) + } + + def run() { + def citys = CommonDatautils.findAllCity() + def cityMap = findCityRange() + def allrets = [] + for (def city : citys) { + def points = cityMap.get(city) + def pointList = InsidePolygon.createPoint3(points); + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "lat", "lng"]) + def rets = findNotInCityDistricts(city, disticts, pointList) + allrets.addAll(rets) + } + ExcelUtils.write("d:\\数据\\小区\\非本城市小区.xlsx", allrets, "经纬度不在此城市中") + } + + def findNotInCityDistricts(city, districts, pointList){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + for(def i=0;i + if(!citys1.contains(c)){ + c = c.replace("州","") + JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") + if(resp.containsKey("districts")){ + def datas = resp.get("districts") + for(def d : datas){ + println c +" "+ d.name+" "+d.level + if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){ + rows << d + }else{ + if(d.level=="city"){ + rows << d + } + } + } + } + } + } + + + +// citys.each{c-> +// println c +// JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") +// if(resp.containsKey("districts")){ +// def datas = resp.get("districts") +// for(def d : datas){ +// println c +" "+ d.name+" "+d.level +// if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){ +// rows << d +// }else{ +// if(d.level=="city"){ +// rows << d +// } +// } +// } +// } +// } + if(rows){ + rows.each {r-> + def obj = new BasicDBObject() + obj.put("level", r.level) + obj.put("center", r.center) + obj.put("center", r.center) + obj.put("polyline", r.polyline) + obj.put("name", r.name) + obj.put("adcode", r.adcode) + obj.put("citycode", r.citycode) + + def city = RegionData.formatCity(r.name) + obj.put("fjw_city", city) + POLYLINECOLL.save(obj) + } + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy similarity index 80% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy index 3c2061a52..994019514 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy @@ -3,6 +3,9 @@ package com.fangjia.script.gj.location import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils @@ -25,28 +28,122 @@ import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern; - -class LocationTest { +/** + * 根据区域围栏修正区域 + */ +@BatchOwner(Watcher.GaoJuan) +class RegionLocationUpdateBatch extends BaseBatch{ static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { -// Date date=new Date(1622802609); -// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); -// String str = format.format(date); -// System.out.println(str); + runBatch(new RegionLocationUpdateBatch(), args); + } - def task = new LocationTest() - task.run() + @Override + public void process(Object args) { + regionUpdate() +// task.run() // task.run2() // task.run3() // task.run4() // task.run5() } + def regionUpdate(){ + def citys = CommonDatautils.findAllCity() + def rets = [] + def index = 0 + for(def city : citys){ + index ++ + def regionPointsMap = findRegionRange(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]); + def datas = findInRegionDistricts(city, districts, regionPointsMap) + println "【${index}】"+city +" 经纬度处理完成" + rets.addAll(datas) + } + ExcelUtils.write("D:\\数据\\小区\\区域纠正.xlsx", rets, "数据") + } + + + def findInRegionDistricts(city, districts, regionPointsMap){ + def index = 0 + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + def regionMap = [:] + regionPointsMap.each{k,v-> + def pointList = InsidePolygon.createPoint3(v); + regionMap.put(k, pointList) + } + def keys = regionPointsMap.keySet() + for(def i=0;i0){ def standardCity = RegionData.formatCity(cityMap.name) if(standardCity){ - saveData(standardCity, m.name, m.ad_code , ret); + def newRegion = RegionData.formatRegion(standardCity, m.name) + saveData(pro, standardCity, m.name, m.ad_code , ret, newRegion); }else{ println cityMap.name +" 没有标准化" } @@ -102,7 +103,7 @@ class RegionPointCrawl { def findCityCode(city){ Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE); def q = null; - if(city.contains("省") || city.endsWith("自治区")){ + if(city.contains("省") || city.endsWith("自治区") || ["上海市","天津市","北京市","重庆市"].contains(city)){ q = new BasicDBObject("province", pattern); }else{ q = new BasicDBObject("name", pattern); @@ -156,18 +157,22 @@ class RegionPointCrawl { } } - def saveData(city, region, code, ret){ + def saveData(pro, city, region, code, ret, formatRegion){ def q = new BasicDBObject() q.put("code", code) def dbo = POLYLINECOLL.findOne(q) if(!dbo){ def o = new BasicDBObject() + o.put("province", pro) o.put("city", city) o.put("region", region) o.put("code", code) o.put("center", ret.center.get(0)) o.put("level", ret.level.get(0)) o.put("polyline", ret.polyline.get(0)); + if(formatRegion){ + o.put("fjw_region", formatRegion) + } POLYLINECOLL.save(o); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index b190a324a..788d1703e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") //广州,佛山 重庆 @@ -204,8 +204,11 @@ public class BeiKeCrawlByChrome { if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl def dbo = findTrade(durl , "贝壳找房") + if (!dbo) { - parse(city, e, region, driver) + parse(city, e, region, driver, null) + }else if(dbo && !dbo.lift){ + parse(city, e, region, driver, dbo._id) } }else{ stop = true; @@ -224,7 +227,7 @@ public class BeiKeCrawlByChrome { return detailUrls } - def parse(city,e, region, driver){ + def parse(city,e, region, driver, id){ try { def detailUrl = e.select(".title a").attr("href") @@ -329,8 +332,11 @@ public class BeiKeCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail - DEAL_PRICE_COLL.save(detail) - + if(id){ + DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail)) + }else{ + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ @@ -459,9 +465,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") -// if (citys.contains(e.text())){ + if (citys.contains(e.text())){ list << [e.text(), href] -// } + } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 07241353c..6c0ce91f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -33,7 +33,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -303,7 +303,9 @@ public class LianjiaCrawlByChrome { detailUrls << durl def dbo = findTrade(durl , "链家在线") if (!dbo) { - parse(city, e, regionText, driver) + parse(city, e, regionText, driver, null) + }else if(dbo && !dbo.lift){ + parse(city, e, regionText, driver, dbo._id) } }else{ stop = true; @@ -326,7 +328,7 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e, regionText, driver){ + def parse(city,e, regionText, driver, id){ try { def price = e.select(".totalPrice .number").text() if(price){ @@ -433,7 +435,12 @@ public class LianjiaCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail - DEAL_PRICE_COLL.save(detail) + if(id){ + DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail)) + }else{ + DEAL_PRICE_COLL.save(detail) + } + } } } @@ -563,9 +570,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") -// if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityList << [e.text(), href] -// } + } } } return cityList diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index f419acf6d..67c803131 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -21,10 +22,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx" addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) +// addFjwPrice(filePath) } @@ -105,12 +107,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (d['B']) { -// def dist = CommonDatautils.findDistrictById(d['B']) -// if(dist){ -// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) - def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['G'] = gxdPrice -// } + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ + def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) +// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) + d['I'] = gxdPrice + } } }) } @@ -129,4 +131,22 @@ class AddPriceTest { } return price } + + + def addFjwPrice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(!d['K']){ + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ +// d['N'] = dist.region + def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd")) + if(dbo){ +// d['K'] = dbo.avg_price + d['K'] = dbo.avg_price_2b + } + } + } + }) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy index 517b6b634..61cc61f88 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy @@ -294,4 +294,8 @@ class AddressUtil { return result } + public static void main(String[] args){ + def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区") + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index fd646bf20..03f3e2b73 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -52,7 +52,40 @@ class InsidePolygon { return list; } - + + + + public static List createPoint3(String points){ + + List alllist = new ArrayList(); + if(points){ + List arrs = new ArrayList(); + if(points.contains("|")){ + String[] array = points.split("\\|"); + arrs.addAll(array) + }else{ + arrs.add(points) + } + + for(def arrStr : arrs){ + List list = new ArrayList(); + String[] array = arrStr.split(";"); + for(int i =0 ;i> getPoints(String points){ String[] array = points.split("-"); List> list = new ArrayList>(); @@ -227,4 +260,16 @@ class InsidePolygon { } return false; } + + + public static def isInArea(lng, lat, pointList){ + def flag = false; + for(def pointArr : pointList){ + flag = InsidePolygon.isPtInPoly(lng, lat, pointArr) + if(flag){ + break; + } + } + return flag; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy index ff11d6991..69c889afe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.temp.shenjian.util +import net.sf.json.JSONObject import org.apache.commons.httpclient.HttpClient import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.RequestEntity import org.apache.commons.httpclient.methods.StringRequestEntity /** @@ -26,4 +28,42 @@ class HttpUtils { } return false } + + + static def doPost2(String url, String jsonString){ + HttpClient httpClient = new HttpClient() + PostMethod postMethod = new PostMethod(url) + String responseMsg = null; + try{ + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + postMethod.setRequestEntity(requestEntity); + httpClient.executeMethod(postMethod); + InputStream is = postMethod.getResponseBodyAsStream(); + BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8")); + StringBuffer sb = new StringBuffer(""); + String str= ""; + while((str = br.readLine()) != null){ + sb.append(str); + } + responseMsg = sb.toString(); + } catch (Exception ex) { + ex.printStackTrace(); + } finally { + postMethod.releaseConnection(); + } + return JSONObject.fromObject(responseMsg); + } + + public String encodeHtml(String html) { + String mh=html; + try { + mh=new String(html.getBytes("iso-8859-1")); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + return mh; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy index c68ba4a85..67858c081 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy @@ -28,7 +28,25 @@ class TradeDealPriceStat { static main(args) { def task = new TradeDealPriceStat() - task.run() +// task.run() + task.deleteRepeat() + } + + def deleteRepeat(){ + def q = new BasicDBObject() + q.put("source", "我爱我家") + def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def set = new HashSet() + for(def c : cursor){ + def tradeDate = DateUtils.formatDateByPattern(c.trade_date, "yyyyMMdd") + def key = c.city +"#"+c.region+"#"+c.district_name+"#"+c.avg_price+"#"+tradeDate + if(set.contains(key)){ + DEAL_PRICE.remove(new BasicDBObject('_id', c._id)) + println c._id + }else{ + set << key + } + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy index b1a045e3f..520cf9f99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20230301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230401", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -38,6 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303") +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index bbc24f035..7b199ce7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220901", "yyyyMMdd") -def endDate = DateUtil.parseDate("20221001", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230201", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230301", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0