diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml
index 811ec290c..cd1282a81 100644
--- a/projects/fangjia-script/pom.xml
+++ b/projects/fangjia-script/pom.xml
@@ -433,11 +433,17 @@
javadbf
1.9.2
-
+
+
+ ru.yandex.clickhouse
+ clickhouse-jdbc
+ 0.2.6
+
+
com.alibaba
fastjson
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportAvgPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportAvgPrice.groovy
index e85c93bdd..fb9821df4 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportAvgPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/ExportAvgPrice.groovy
@@ -30,12 +30,12 @@ public class ExportAvgPrice {
DSCollection dataStoreConnection = null;
static main(args) {
def task = new ExportAvgPrice();
- def weekends = ["2019-1-1","2019-2-1","2019-3-1","2019-4-1","2019-5-1","2019-6-1","2019-7-1","2019-8-1", "2019-9-1","2019-10-1","2019-11-1","2019-12-1","2020-1-1","2020-2-1","2020-3-1","2020-4-1","2020-5-1","2020-6-1","2020-7-1","2020-8-1", "2020-9-1","2020-10-1","2020-11-1","2020-12-1","2021-1-1","2021-2-1","2021-3-1","2021-4-1","2021-5-1","2021-6-1","2021-7-1","2021-8-1", "2021-9-1","2021-10-1"];
- def citys = ["温州"];//findAllCity();
-// task.exportAllCity(citys, weekends, "e:\\全国城市季度均价.csv");
- task.exportRegion(citys, weekends, "d:\\温州区域季度均价.csv");
+ def weekends = ["2023-04-01","2023-03-01","2023-02-01","2023-01-01","2022-12-01","2022-11-01","2022-10-01","2022-09-01","2022-08-01","2022-07-01","2022-06-01","2022-05-01","2022-04-01","2022-03-01","2022-02-01","2022-01-01","2021-12-01","2021-11-01","2021-10-01","2021-09-01","2021-08-01","2021-07-01","2021-06-01","2021-05-01","2021-04-01"]
+ def citys = ["上海"];//findAllCity();
+// task.exportAllCity(citys, weekends, "D:\\数据\\小区\\全国城市季度均价.csv");
+// task.exportRegion(citys, weekends, "D:\\数据\\小区\\区域季度均价.csv");
// task.exportBlock(citys, weekends, "e:\\区域板块季度均价.csv");
-// task.exportDistrict(citys, weekends, "e:\\小区季度均价.csv");
+ task.exportDistrict(citys, weekends, "D:\\数据\\小区\\小区季度均价.csv");
// task.exportRegionPrice("江苏");
}
@@ -182,6 +182,7 @@ public class ExportAvgPrice {
q.put("visible", 0);
q.put("cat", "district");
q.put("city", city);
+// q.put("name", "龙柏二村")
DSCursor cursor = getDataStoreConnection().find(q).sort(new BasicDBObject('bscore', -1));
Iterator iters = cursor.iterator()
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy
new file mode 100644
index 000000000..0a601a6ec
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy
@@ -0,0 +1,36 @@
+package com.fangjia.script.ccbevaluation
+
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+
+def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
+def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
+DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
+
+for(def d : datas){
+ def map = new BasicDBObject()
+ map.city = "深圳"
+ map.district_name = d['D']
+ map.region = d['B']
+ def roomStr = d['E']
+ if(roomStr){
+ def arrs = roomStr.split("-")
+ map.room = arrs[0].toString().toInteger()
+ map.hall = arrs[1].toString().toInteger()
+ map.toilet = arrs[3].toString().toInteger()
+ }
+
+ map.area = d['F'].toString().toDouble()
+ map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
+ map.avg_price = (int)(d['H'].toString().toDouble()*10000)
+ def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31")
+ map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd")
+ map.source = "深房家族"
+ map.create_date = new Date()
+ map.m_date = new Date()
+ DEAL_PRICE_COLL.save(map)
+}
+
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
index e00fdeb41..5e7d45516 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy
@@ -67,50 +67,50 @@ class BigdataDealBatch extends BaseBatch{
}
println "curDate:"+curDate
-// for(def city : citys){
-//// if(!ccbCitys.contains(city)){
-// addDealPriceMatch(city);
-//// }
-// }
- def rows = []
- def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
- def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
- def index = 0
for(def city : citys){
- index ++
- def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
- def ppeDists = [:]
-
- def statMap = [:]//getDistrictStatByCity(city, weekend)
- if(city=="上海"){
- def youDataTrade = getYouDaData(statMap)
- def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
- rows.addAll(youdaDatas)
- println city +" "+youdaDatas.size()
- }
-
- def trades = getCityData(city, statMap, ppeDists)
- if(trades){
- def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
- rows.addAll(rets)
- println "【${index}】"+city +" 成交共:"+rets.size()
-
- def cityDatas = []
- cityDatas << headers
- cityDatas.addAll(rows)
- }
-
+// if(!ccbCitys.contains(city)){
+ addDealPriceMatch(city);
+// }
}
-
- def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
- if(CommonDatautils.isWindow()){
- fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
- }
-
- FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
- println "txt 生成 " + rows.size()
-
- saveToPgSql(rows)
+// def rows = []
+// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
+// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
+// def index = 0
+// for(def city : citys){
+// index ++
+// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
+// def ppeDists = [:]
+//
+// def statMap = [:]//getDistrictStatByCity(city, weekend)
+// if(city=="上海"){
+// def youDataTrade = getYouDaData(statMap)
+// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
+// rows.addAll(youdaDatas)
+// println city +" "+youdaDatas.size()
+// }
+//
+// def trades = getCityData(city, statMap, ppeDists)
+// if(trades){
+// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
+// rows.addAll(rets)
+// println "【${index}】"+city +" 成交共:"+rets.size()
+//
+// def cityDatas = []
+// cityDatas << headers
+// cityDatas.addAll(rows)
+// }
+//
+// }
+//
+// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
+// if(CommonDatautils.isWindow()){
+// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
+// }
+//
+// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
+// println "txt 生成 " + rows.size()
+//
+// saveToPgSql(rows)
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
index d579b79f1..cdad2a607 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
@@ -375,12 +375,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
- def month = 202302
+ def month = 202303
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
- def rets = findPeGteThirdParty2(city, 202303)
+ def rets = findPeGteThirdParty2(city, 202304)
def total = 0
println city +" "+rets.size()
if(rets){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy
index 4c9574fd8..5c2e8c26c 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy
@@ -33,8 +33,8 @@ class FjwPricePkCcbPrice {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
- task.addFjwPrice()
-// task.addCcbPrice()
+// task.addFjwPrice()
+// task.addCcbPrice()//oracle
// task.addCcbPriceFromGuided()
// task.addMarketPrice()
@@ -42,21 +42,36 @@ class FjwPricePkCcbPrice {
// task.exportFjwPriceAndCcbPrice()
// task.addNewDistPrice()
-// task.addCcbPrice2()
-// task.addCcbPrice3()
+ task.addCcbPrice2()//按id查
+// task.addCcbPrice3()//按city,region,name查
// task.standRegion()
+// task.addGuidPrice()
}
+
def addCcbPrice2(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
- ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d ->
- if(d['R']){
- def map = findAvgPrice(d['R'], cityCodeMap.get(d['S']))
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d ->
+ if(d['C']){
+ def map = findAvgPrice(d['C'], cityCodeMap.get(d['E']))
if(map){
- d['AG'] = map.avg_price
- d['AH'] = map.calc_type
- d['AI'] = map.credit_value
- d['AJ'] = map.month
+ d['K'] = map.avg_price
+ d['L'] = map.calc_type
+// d['AI'] = map.credit_value
+ d['M'] = map.month
+ }
+ }
+ })
+ }
+
+ def addGuidPrice(){
+ def cityCodeMap = CcbCommonUtils.findCityCodeValue()
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_230421.xlsx", { d ->
+ if(d['M']){
+ def map = findGuidePrice(d['C'], d['E'])
+ if(map){
+ d['M'] = map.avg_price
+ d['N'] = map.date.toString()
}
}
})
@@ -474,4 +489,14 @@ class FjwPricePkCcbPrice {
})
return map
}
+
+ def findGuidePrice(def commid, def city){
+ def sql = "SELECT gujia_bds_drc_prc,gujia_drc_dt from base.t0831_guided_price_gj_h where urbn_nm='${city}' and gujia_bds_id='${commid}' ORDER BY gujia_drc_dt desc limit 1"
+ def map = [:]
+ pgsqlInstance.eachRow(sql, { r ->
+ map.avg_price = r.gujia_bds_drc_prc
+ map.date = r.gujia_drc_dt
+ })
+ return map
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy
index 716114623..b78a31710 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
- def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-21")
+ def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-26")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//更新静态数据表
- updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
+// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy
index 5474a893c..db8fef0a1 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy
@@ -71,33 +71,33 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
//
//})
-def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
-def allCitys = CcbCommonUtils.citys_all
-def cityCodeMap = CcbCommonUtils.findCityCodeValue()
-def index = 0
-for(def c : allCitys){
- def cityCode = cityCodeMap.get(c)
- def count = null;
- index++
- for(def r : countCitys){
- if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
- count = r['B']
- break
- }
- }
- println index +" "+ c+" "+count
-}
-
-def findCcbPriceCredit(sqlInstance, cityid, commid){
- def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
- def map = [:]
- sqlInstance.eachRow(sql,{r->
- map.month = r.year_month
- map.credit = r.CONFIDENCE
- map.price = r.avg_price
- });
- return map
-}
+//def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
+//def allCitys = CcbCommonUtils.citys_all
+//def cityCodeMap = CcbCommonUtils.findCityCodeValue()
+//def index = 0
+//for(def c : allCitys){
+// def cityCode = cityCodeMap.get(c)
+// def count = null;
+// index++
+// for(def r : countCitys){
+// if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
+// count = r['B']
+// break
+// }
+// }
+// println index +" "+ c+" "+count
+//}
+//
+//def findCcbPriceCredit(sqlInstance, cityid, commid){
+// def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
+// def map = [:]
+// sqlInstance.eachRow(sql,{r->
+// map.month = r.year_month
+// map.credit = r.CONFIDENCE
+// map.price = r.avg_price
+// });
+// return map
+//}
def findThirdPriceBySource(sqlInstance, cityid, commid){
def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" +
@@ -1314,6 +1314,24 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
+def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
+ExcelUtils.multiThreadingModify(fileName, { d ->
+ def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
+ if(count && count==1){
+ def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
+ d['E'] = dist.city
+ }
+
+// def dist = CommonDatautils.findDistrictById(d['A'])
+// if(dist){
+// d['B'] = dist.region
+// d['C'] = dist.name
+// d['J'] = dist.status
+// }
+// def newRegion = RegionData.formatRegion(d['A'], d['B'])
+// d['F'] = newRegion
+},0)
+
//ExcelUtils.modify(fileName, {d->
// if(d['A']){
// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'"
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy
index 076edf823..3c0226e81 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy
@@ -92,13 +92,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
- rets << ["城市","12月总量","23年1月总量","23年2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
+ rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
- def startDate = "2022-12-01"
- def date = "2023-01-01"
- def endDate = "2023-02-01"
- def newDate = "2023-03-01"
+ def startDate = "2023-01-01"
+ def date = "2023-02-01"
+ def endDate = "2023-03-01"
+ def newDate = "2023-04-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy
index 9ed8300cc..5a38c237e 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
-// cityRun(citys, startDate, endDate, cityCodeMap)
-// regionRun(citys, startDate, endDate, cityCodeMap)
+ cityRun(citys, startDate, endDate, cityCodeMap)
+ regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy
index ce4355ead..591cd3b69 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy
@@ -1012,7 +1012,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgThirdPartyPrice2(price_date, new_dists){
- def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
+ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
@@ -1184,7 +1184,7 @@ class ExportDistrictRongHe extends BaseBatch{
});
}
- def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
+ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
@@ -1273,7 +1273,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgCommunityPdescribeGj(month){
- def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
+ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
@@ -1381,7 +1381,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgGuideStatistics(time){
- def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
+ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
@@ -1417,7 +1417,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataSupplierStatistics(time){
- def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
+ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
@@ -1532,7 +1532,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataModelingBase(time){
- def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
+ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
index cae3e60b2..b7bf2feb6 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
@@ -622,7 +622,10 @@ class CommonDatautils {
public static def findDistrictCount(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
- q.put("city", city);
+ if(city){
+ q.put("city", city);
+ }
+
if(region){
q.put("region", region);
}
@@ -639,7 +642,10 @@ class CommonDatautils {
public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
- q.put("city", city);
+ if(city){
+ q.put("city", city);
+ }
+
if(region){
q.put("region", region);
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy
index 2da506d11..917ddfc5e 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy
@@ -25,7 +25,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
-// task.regionPriceTrendRun()
+ task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
index 89347b833..e26dc05dd 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
-def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv")
+def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv")
def count0 = 0
def count1 = 0
def count2 = 0
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy
index 7ed37c86c..8ccb30a9a 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy
@@ -24,7 +24,13 @@ class AddAvgpriceBatch {
static main(args) {
def task = new AddAvgpriceBatch();
- task.run1();
+// task.run1();
+ task.test()
+ }
+
+ def test(){
+ //发重算均价命令
+ addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd'))
}
def run(){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy
index cdbeaefd7..805c41944 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy
@@ -2,6 +2,7 @@ package com.fangjia.script.gj
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
@@ -263,8 +264,7 @@ class FiledStrand {
if(all_area){
def v = getValue(all_area)
if(v>=1000 && v<=10000000){
-
- return v.toString()
+ return v.toString().toDouble().toInteger()
}
}
return null
@@ -273,7 +273,7 @@ class FiledStrand {
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
- if(v>=1 && v<=100000){
+ if(v>5 && v<=100000){
try {
return v.toString().toInteger();
}catch(Exception e){
@@ -284,6 +284,70 @@ class FiledStrand {
return null
}
+ static def processCarSiteDesc(car_site){
+ if(car_site && car_site.size()<=20){
+ return car_site.toString()
+ }
+ return null
+ }
+
+
+ static def processPropertyLife(car_site_count){
+ def v = getValue(car_site_count.toString());
+ if(v){
+ if([40,50,70].contains(v)){
+ try {
+ return v.toString()+"年";
+ }catch(Exception e){
+ println e.getMessage()
+ }
+ }
+ }
+ return null
+ }
+
+ static def processAlias(esVal, name, alias){
+ if(esVal){
+ if(esVal!=name){
+ if(!alias){
+ alias = []
+ }
+ def arr = alias.split(",")
+ for(def s : arr){
+ if(!alias.contains(s)){
+ alias << esVal
+ }
+ }
+
+ if(alias){
+ return alias
+ }
+ }
+ }
+ return null
+ }
+
+ static def processAddress(def esAddress, def city){
+ if(esAddress){
+ def formatAddress = AddressUtil.formatAddress(city, esAddress)
+ if(formatAddress && isRoad(formatAddress)){
+ return formatAddress
+ }
+ }
+ return null
+ }
+
+ static def isRoad(def s){
+ def pattern = Pattern.compile("[路|街|大道|巷]");
+ def matcher = pattern.matcher(s)
+ if(matcher.find()){
+ return true;
+ }
+ return false
+ }
+
+
+
static def addBuildAreaToData(def dist, def updateFile){
def build_area = null;
if(dist.build_area){
@@ -324,18 +388,12 @@ class FiledStrand {
return updateFile;
}
- public static def processCubagePercent(cubage_percent, estate_type2){
+ public static def processCubagePercent(cubage_percent){
if(cubage_percent){
def v = getValue(cubage_percent)
if(v){
- if(estate_type2 == "别墅"){
- if(v>=0.3 && v<=1){
- return v.toString()
- }
- }else{
- if(v>1 && v<=10){
- return v.toString()
- }
+ if(v>=0.3 && v<=10){
+ return v.toString()
}
}
}
@@ -418,26 +476,26 @@ class FiledStrand {
estateCharege = estateCharege.toDouble()
if(estateType2=="普通住宅"){
if(city=="上海" || city=="北京" || city=="广州" || city=="杭州"){
- if(estateCharege>=0.6 && estateCharege<=3.5){
+ if(estateCharege>=0.3 && estateCharege<=40){
return estateCharege
}else{
return null;
}
}else{
- if(estateCharege>=0.2 && estateCharege<=3.5){
+ if(estateCharege>=0.2 && estateCharege<=20){
return estateCharege
}else{
return null;
}
}
}else if(estateType2=="别墅"){
- if(estateCharege>=3){
+ if(estateCharege>=1){
return estateCharege;
}else{
return null;
}
}else if(estateType2=="商住"){
- if(estateCharege>=4){
+ if(estateCharege>=2){
return estateCharege;
}else{
return null;
@@ -454,13 +512,13 @@ class FiledStrand {
if(household_count){
household_count = household_count.toInteger()
if(estateType2=="普通住宅" || estateType2=="商住"){
- if(household_count>=50 && household_count<=100000){
+ if(household_count>=15 && household_count<=100000){
return household_count;
}else{
return null;
}
}else if(estateType2=="别墅"){
- if(household_count>=20 && household_count<=100000){
+ if(household_count>=10 && household_count<=100000){
return household_count;
}else{
return null;
@@ -470,6 +528,20 @@ class FiledStrand {
return null;
}
+ static def processLoudongCount(loudongCount){
+ def v = getValue(loudongCount.toString());
+ if(v){
+ if(v>=1 && v<=10000){
+ try {
+ return v.toString().toInteger();
+ }catch(Exception e){
+ println e.getMessage()
+ }
+ }
+ }
+ return null
+ }
+
static def processFinishBuildingDate(def finishBuildingDate){
if(finishBuildingDate){
Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}");
@@ -493,6 +565,39 @@ class FiledStrand {
}
+ static def processCompany(val){
+ if(val){
+ if(val.length()>20){
+ return null
+ }else{
+ if(val.contains("无")){
+ return null
+ }
+ def fiterChars = [",",".","?","/","数据",'-','?',' ',' ']
+ for(def s : fiterChars){
+ val = val.replace(s, '')
+ }
+
+ if(val.length()<1){
+ return null
+ }
+
+ if(val.toString().isNumber()){
+ return null
+ }
+
+ Pattern pattern = Pattern.compile("\\d{4}年");
+ if (pattern.matcher(val).matches()) {
+ return null
+ }
+ return val
+ }
+ }
+
+ return null;
+ }
+
+
def converList(def dbCursor){
def list = []
for(def dbo : dbCursor){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
index 210802c58..e4e5016ec 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
@@ -143,17 +143,8 @@ class OpenApiUtils {
def result = queryMaxSimilarAddress(city, name)
if(result){
for(def ret : result){
- if(ret && ret.lat){
- try{
- def gmap = GeocodeUtils.getGaodeToBaidu(ret.lng.toDouble(), ret.lat.toDouble())
- ret.lng2 = gmap.get("lng")
- ret.lat2 = gmap.get("lat")
- ret.estateType = ret.type
- ret.source = "高德"
- }catch (Exception e){
-
- }
- }
+ ret.estateType = ret.type
+ ret.source = "高德"
}
}
return result
@@ -161,11 +152,17 @@ class OpenApiUtils {
public static def queryMaxSimilarAddress(String city, String address, boolean isFullMatch = false){
- if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
+// if(StringUtils.isEmpty(city)){
+// return null
+// }
+ if(StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","")
- String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
+// String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
+ //key:8796b9dfb3889b57c38e2fff1b7e51bd 花钱买的
+ String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&city=" + city + "&output=json&offset=20&page=1&key=8796b9dfb3889b57c38e2fff1b7e51bd&extensions=all&citylimit=true"//citylimit限制城市
+
List poiList = null;
com.alibaba.fastjson.JSONObject result = null;
@@ -553,6 +550,35 @@ class OpenApiUtils {
return null;
}
+ static def similarRank(proCode, cityCode, regionCode, name, area, floor, height){
+ def url = "http://localhost:8080/fangjia-open/house/similar?aboveFloorAmount=${height}&buildingName=${name}&city=${cityCode}&district=${regionCode}&" +
+ "propertyTypeCode=01&province=${proCode}&floorNumber=${floor}&strctureArea=${area}"
+ println url
+ String content = new HttpRequest().get(url, "utf-8");
+ def dto = []
+ if(content){
+ def doc = new JsonSlurper().parseText(content)
+ if(doc && doc.code ==200 && doc.result){
+ def dtos = doc.result
+ return dtos
+ }
+ }
+ return null;
+ }
+
+ static def financialRating(city, address){
+ def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3"
+ String content = new HttpRequest().get(url, "utf-8");
+ if(content){
+ def doc = new JsonSlurper().parseText(content)
+ if(doc && doc.code ==0 && doc.result){
+ def dtos = doc.result
+ return dtos
+ }
+ }
+ return null;
+ }
+
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy
index 52f2ea980..0842f9b06 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
- static def statDate = "2023-01-01"
+ static def statDate = "2023-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
-// task.esfParser("D:\\数据\\友达\\2023年2月\\查询1.DBF");//友达二手房存入mongo
+// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy
index 2817d78ee..73ea03b09 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy
@@ -3,6 +3,7 @@ package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
+import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
@@ -12,29 +13,106 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
-def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
-def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
+//def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
+//def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
+//Calendar calendar = Calendar.getInstance()
+//calendar.setTime(startDate)
+//def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
+//def rMap = [:]
+//def index = 0
+//OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
+// index++
+// def day = doc.c_date.format("yyyy-MM-dd")
+// def count = rMap.get(day)
+// if (!count) {
+// rMap.put(day, 1)
+// } else {
+// rMap.put(day, count + 1)
+// }
+// if(index%10000==0){
+// println index+" "+new Date().toString()
+// }
+//}
+//
+//rMap.each {k,val->
+// println k+" "+val
+//}
+
+//def startDate = DateUtil.parseDate("20230416", "yyyyMMdd")
+//def endDate = DateUtil.parseDate("20230418", "yyyyMMdd")
+//Calendar calendar = Calendar.getInstance()
+//calendar.setTime(startDate)
+//def count = 0
+//def hashSet = []
+//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
+//while (startDate.before(endDate)) {
+// calendar.add(Calendar.DATE, 1)
+// def nextDate = calendar.getTime()
+// count++
+// def total = 0
+// OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
+// total++
+// def result = JSONObject.fromObject(doc.res_map.result)
+//// if (!hashSet.contains(result)) {
+// def item = [doc._id.toString(), "neiping2017@czbank", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
+// data << item
+// hashSet << result
+// println total+" "+ nextDate
+//// } else {
+//// println("已存在")
+//// }
+// }
+// if (count % 7 == 0) {
+// hashSet.clear()
+// }
+//
+// startDate = nextDate
+//
+//}
+//
+////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
+//ExcelUtils.write("D:\\浙商调用详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
+
+
+
+
+def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
+def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
-def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
-def rMap = [:]
-def index = 0
-OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
- index++
- def day = doc.c_date.format("yyyy-MM-dd")
- def count = rMap.get(day)
- if (!count) {
- rMap.put(day, 1)
- } else {
- rMap.put(day, count + 1)
- }
- if(index%10000==0){
- println index+" "+new Date().toString()
- }
-}
+def count = 0
+def hashSet = []
+def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
-rMap.each {k,val->
- println k+" "+val
+OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
+ def result = JSONObject.fromObject(doc.res_map.result)
+ def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
+ data << item
+ hashSet << result
}
+//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
+ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
+
+
+//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd")
+//def endDate = DateUtil.parseDate("20230506", "yyyyMMdd")
+//Calendar calendar = Calendar.getInstance()
+//calendar.setTime(startDate)
+//def count = 0
+//def hashSet = []
+//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
+//
+//OPEN_URI_LOG.find(and(eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
+// def result = JSONObject.fromObject(doc.res_map.result)
+// def item = [doc._id.toString(), doc.user_id, doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
+// data << item
+// hashSet << result
+// count++
+// println count
+//}
+//
+//
+////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
+//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy
index b28cc2770..269b5be9e 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy
@@ -33,11 +33,14 @@ class AnjukeDistrictMatch {
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
+// def name = "陆家嘴滨江中心N2座"
+// name = name.replaceAll("-?\\w+(号写字楼|号楼|楼|座|栋)","")
+// println name
def task = new AnjukeDistrictMatch();
// task.run();
// task.runCsv();
// task.updateCredit()
-// task.matchCommunity()//查询es小区数据
+ task.matchCommunity()//查询es小区数据
// task.run1()
// task.updateGreenRateField()//根据第三方数据填补库中字段
// task.updteGreenRateFieldByCsv()
@@ -50,7 +53,7 @@ class AnjukeDistrictMatch {
// task.updteBuildAreaFieldByCsv()
// task.updadteAllAreaFieldByCsv()
// task.updatePlotRateFieldByCsv()
- task.updateFinishDateFieldByCsv()
+// task.updateFinishDateFieldByCsv()
}
def updateCredit(){
@@ -728,16 +731,17 @@ class AnjukeDistrictMatch {
def run(){
- def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
- def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\全国绿化率.xlsx"
+// def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
+ def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)0406.xlsx"
def index = 0
ExcelUtils.multiThreadingModify(filePath,{row->
index++
if(row['A'] && row['B']){
def city = row['A']
- def name = row['C']
- def region = row['B']
+ def name = row['B']
+ def region = null
+// def region = row['B']
if(name && name.contains("·")){
name = name.replace("·", "")
@@ -785,27 +789,27 @@ class AnjukeDistrictMatch {
row['AC'] = jsonObject.get("credit")
}
}
- println index +" "+row['Y']
-// if(row['G']){
-// def dbo = CommonDatautils.findDistrictById(row['G'])
-// if(dbo){
-// row['H'] = dbo.estate_type2
-// row['I'] = dbo.name_cat
-//
-// def count = findBigdataCommunity(row['G'])
-// if(count){
-// row['E'] = "是"
-// }
-// }
-// }
+// println index +" "+row['Y']
+ if(row['G']){
+ def dbo = CommonDatautils.findDistrictById(row['G'])
+ if(dbo){
+ row['H'] = dbo.estate_type2
+ row['I'] = dbo.name_cat
+
+ def count = findBigdataCommunity(row['G'])
+ if(count){
+ row['E'] = "是"
+ }
+ }
+ }
}
})
}
def matchCommunity(){
- def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)3.24_2.xlsx"
+ def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)0406-1.xlsx"
ExcelUtils.modify(filePath, { row ->
- if (row['A'] && row['B'] && row['I']) {
+ if (row['A'] && row['B'] && !row['O']) {
def rets = findAllEsData(row['A'], row['B'])
if(rets){
def dbo = rets[0]
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy
index 231a930c8..55ce0ed8f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy
@@ -480,7 +480,7 @@ class OfficeProcess {
}
def checkRegion(){
- def citys = ["北京","广州","重庆","天津","成都","南京"]//"上海","深圳",
+ def citys = ["杭州","苏州"]//"上海","深圳","北京","广州","重庆","成都",
for(def city : citys){
def regionPointsMap = findRegionRange(city)
def addDatas = findAddDatasByClickhouse(city)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy
index 6b07a6159..62666390b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy
@@ -16,7 +16,140 @@ class FieldStat {
// task.run()
// task.run1()
// task.run2()
- task.run3()
+// task.run3()
+// task.offircRun()
+ task.distRun()
+ task.shopRun()
+ }
+
+ def distRun(){
+ def citys = CommonDatautils.findAllCity()
+ def cityMap = [:]
+ def fileds = ["region","name","alias","address","addr_alias","estate_type2","build_type","block","loop_line_info","street_community","household_count","curr_household_count","virescence_percent","cubage_percent","all_area","build_area","estate_charge","car_site","car_site_count","developer","estate_company","finish_building_date","live_date","start_building_date","open_date","property_life","floor_info","fitment","fitment_materials","build_structure","traffic","facilities","characteristic","sell_address","relate_info","description","total_price","mobile","building_status","licence","build_designer","garden_designer","business_quarter","bay_area","payment_pattern","land_plan_license","prj_plan_license","land_use_license","cons_license","elevators_count","build_date","divisible","status","metro_info","metro_names","cons_company","sells_company","contact_people","parking_fee","delivery_level","fitment_level","land_car_site","underground_car_site","avg_price","sell_price","standard_area","pics","loudong_count","committee","house_structure","usage_card","mainTypeArea"]
+ for(def c : citys){
+ def dbo = findCoverPrecent(c)
+ if(dbo && dbo.containsField("district_detail")){
+ def officeDetail = dbo.district_detail
+ cityMap.put(c, officeDetail)
+ }else{
+ cityMap.put(c, [:])
+ }
+ }
+
+ def rets = []
+ rets << ["城市","数量","区域","名称","别名","地址","地址别名","物业类型","建筑类型","板块","环线位置","街道社区","户数","当前户数","绿化率","容积率","占地面积","建筑面积","物业费","停车位","停车位数","开发商","物业公司","竣工时间","入住时间","开工时间","开盘时间","产权年限","楼层情况","装修情况","装修材料","建筑结构","交通情况","周边配","小区特征","售楼地址","相关信息","描述","总价","电话","建筑状态","预售许可证","建筑单位","园艺设计单位","商圈","户型面积","支付方式","用地规划许可证","工程规划许可证","土地使用证","施工许可证","电梯数量","建造年代","是否可分割","状态","地铁沿线","地铁站名","施工单位","销售企业","联系人","停车费","交付标准","装修标准","地上车位数","地下车位数","均价","出售价格","标准层面积","图片","楼栋总数","居委会","房屋结构","土地使用权证号","主力户型"]
+ def ret = []
+ ret << "city"
+ ret << "count"
+ for(def f : fileds){
+ ret << f
+ }
+ rets << ret
+ cityMap.each {k,v->
+ def row = []
+ row << k
+ row << CommonDatautils.findAllDistrictCount("district", k)
+ for(def filed : fileds){
+ if(v){
+ def val = v.get(underlineToCamel(filed))
+ if(val){
+ row << val.toDouble()*100
+ }else{
+ row << ""
+ }
+ }
+ }
+ rets << row
+ }
+
+ ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "小区")
+ }
+
+ def shopRun(){
+ def citys = CommonDatautils.findAllCity()
+ def cityMap = [:]
+ def fileds = ["region","name","alias","address","business_quarter","property_status","rent_type","lease_term","payment_pattern","sell_price","household_count","mobile","floor","floor_info","bay_area","finish_building_date","live_date","divisible","unified_manage","car_site_count","car_site","all_area","build_area","estate_charge","developer","estate_company","metro_info","metro_names","facilities","property_info","traffic","surrounding","business_status","url","description","open_date","sell_address","block","height","loop_line_info","cubage_percent","virescence_percent","characteristic","sells_company","licence","build_designer","cons_company","property_life","insidepos","delivery_level","start_building_date","land_plan_license","prj_plan_license","land_use_license","cons_license","fitment","build_structure","enter_company","build_type","elevators_count","estate_type2","status","standard_area","relate_info","garden_designer","fitment_level","build_date","fitment_materials","rent_price","out_pics","pics","land_car_site","underground_car_site","addr_alias"]
+ for(def c : citys){
+ def dbo = findCoverPrecent(c)
+ if(dbo && dbo.containsField("shop_detail")){
+ def officeDetail = dbo.shop_detail
+ cityMap.put(c, officeDetail)
+ }else{
+ cityMap.put(c, [:])
+ }
+ }
+
+ def rets = []
+ rets << ["城市","数量","区域","名称","别名","地址","商圈","建筑状态","租售类型","租赁年限","付款方式","出售价格","总户数","电话","建筑楼层","楼层状况","开间面积","竣工时间","开业时间","可分割","统一管理","车位数量","车位描述","占地面积","建筑面积","物业费","开发商","物业公司","地铁沿线","地铁站名","配套设施","物业概况","交通情况","周边情况","经营业态","来源url","描述","开盘时间","售楼地址","板块","楼高","环线位置","容积率","绿化率","商业特色","销售企业","销售许可证","设计单位","施工单位","产权年限","交通位置","交付标准","开工时间","用地规划许可证","工程规划许可证","土地使用证","施工许可证","装修","建筑结构","入驻企业","建筑类型","电梯数量","物业类型","状态","标准层面积","相关信息","景观设计单位","装修标准","建造年代","建材装修","出租价格","外景图","其它图","地上车位数","地下车位数","地址别名"]
+ def ret = []
+ ret << "city"
+ ret << "count"
+ for(def f : fileds){
+ ret << f
+ }
+ rets << ret
+ cityMap.each {k,v->
+ def row = []
+ row << k
+ row << CommonDatautils.findAllDistrictCount("shop", k)
+ for(def filed : fileds){
+ if(v){
+ def val = v.get(underlineToCamel(filed))
+ if(val){
+ row << val.toDouble()*100
+ }else{
+ row << ""
+ }
+ }
+ }
+ rets << row
+ }
+
+ ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "商铺")
+ }
+
+ def offircRun(){
+ def citys = CommonDatautils.findAllCity()
+ def cityMap = [:]
+ def fileds = ["region","name","alias","latLng","latLng2","address","addrAlias","bedKind","estateType2","buildType","block","businessQuarter","loopLineInfo","licence","sideset","realUsePercent","buildDesigner","consCompany","estateCharge","carSiteCount","carSite","undergroundCarSite","landCarSite","parkingFee","sight","propertyLife","lobbyHeight","standardHeight","clearHeight","standardArea","foreign","register","lightingVentilation","conditionerTime","","elevatorsCount","personElevatorsCount","goodsElevatorCount","openDate","finishBuildingDate","liveDate","allArea","buildArea","cubagePercent","virescencePercent","developer","estateCompany","networkWay","fitmentLevel","buildStructure"]
+ for(def c : citys){
+ def dbo = findCoverPrecent(c)
+ if(dbo && dbo.containsField("office_detail")){
+ def officeDetail = dbo.office_detail
+ cityMap.put(c, officeDetail)
+ }else{
+ cityMap.put(c, [:])
+ }
+ }
+
+ def rets = []
+ rets << ["城市","数量","区域","名称","名称别名","经纬度(百度)","经纬度(高德)","地址","地址别名","写字楼等级","写字楼类型","建筑类别","板块","商圈","环线位置","预售许可证","是否可分割","得房率","设计单位","施工单位","物业费","停车位数","停车描述","地上车位数","地下车位数","车位租金","开间面积","产权年限","大堂层高","标准层高","净高","标准层面积","是否涉外","是否可注册","空调","空调开放时间","空调费用","电梯数量","客梯数","货梯数","开盘时间","竣工时间","入住时间","占地面积","建筑面积","容积率","绿化率","开发商","物业公司","网络","装修状况","结构"]
+ def ret = []
+ ret << "城市"
+ ret << "数量"
+ for(def f : fileds){
+ ret << f
+ }
+ rets << ret
+ cityMap.each {k,v->
+ def row = []
+ row << k
+ row << CommonDatautils.findAllDistrictCount("office", k)
+ for(def filed : fileds){
+ if(v){
+ def val = v.get(filed)
+ if(val){
+ row << val.toDouble()*100
+ }else{
+ row << ""
+ }
+ }
+ }
+ rets << row
+ }
+
+ ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "写字楼")
+
}
def run3(){
@@ -136,4 +269,24 @@ class FieldStat {
}
return obj
}
+
+
+ public static String underlineToCamel(String param){
+ if (param==null||"".equals(param.trim())){
+ return "";
+ }
+ int len=param.length();
+ StringBuilder sb=new StringBuilder(len);
+ for (int i = 0; i < len; i++) {
+ char c=param.charAt(i);
+ if (c=="_"){
+ if (++i
+ if (d['A']) {
+ def results = com.fangjia.script.gj.OpenApiUtils.financialRating(d['A'], d['B']+d['C'])
+ if(results){
+ d['D'] = results.consumptionRating
+ }
+ }
+ })
+ }
+
+ def similarRunTest(){
+
+ ExcelUtils.modify("D:\\数据\\江南农商行\\住宅信息V2-差异案例(反馈).xlsx", { d ->
+ if(d['A']){
+ def cityCode = d['U'].toString().toDouble().toInteger()
+ def proCode = d['V'].toString().toDouble().toInteger()
+ def regionCode = d['T'].toString().toDouble().toInteger()
+ def name = d['D']
+ def area = d['F']
+ def floor = d['H'].toString().toDouble().toInteger()
+ def height = d['G'].toString().toDouble().toInteger()
+ def results = com.fangjia.script.gj.OpenApiUtils.similarRank(proCode, cityCode,regionCode, name, area, floor, height)
+ if(results){
+ def str = ""
+ for(def r : results){
+ str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"\n"
+ }
+ d['Y'] = str
+ }
+ }
+ })
+ }
+
+ def rankRunTest(){
def dists = CommonDatautils.findAllDistrictByCity("上海","district", ["city","region","name"])
//def dists = CommonDatautils.findDistrictByCityAndRegions("上海","district", ["浦东"], ["city","region","name"])
def rets = createCityDatas(dists)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy
new file mode 100644
index 000000000..d32f3e459
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy
@@ -0,0 +1,417 @@
+package com.fangjia.script.gj.district
+
+import com.fangjia.script.BaseBatch
+import com.fangjia.script.BatchOwner
+import com.fangjia.script.Watcher
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.common.DataHandleUtils
+import com.fangjia.script.gj.FiledStrand
+import com.fangjia.script.lixue.util.DBUtils
+import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.ElasticUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.fangjia.script.util.OpenApiClient
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.rabbitmq.client.impl.AMQImpl
+import org.bson.types.ObjectId
+
+import java.util.concurrent.CopyOnWriteArrayList
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+import java.util.regex.Pattern
+
+/**
+ * 小区字段入库(自动入库)
+ */
+@BatchOwner(Watcher.GaoJuan)
+class DistrictFieldAddBatch extends BaseBatch{
+
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
+
+ DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018)
+ //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
+
+ //,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"
+ def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源
+
+ //"alias":"alias",
+ def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段
+
+ static main(args) {
+ runBatch(new DistrictFieldAddBatch(), args)
+ }
+
+ @Override
+ void process(Object args) {
+ def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京"
+
+
+ def analysRets = []
+ for(def c : citys){
+ def totalCount = CommonDatautils.findAllDistrictCount("district", c)
+// addFieldFromRonghe(c, totalCount)
+ analysRets = addFieldFromMatch(c, totalCount, analysRets)
+ }
+ println analysRets.toString()
+
+ ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
+ }
+
+ def addFieldFromMatch(c, totalCount, analysRets){
+ sourceMap.each { seaweedmapSource, esWebSource ->
+ def esRets = findEsDatas(c, esWebSource)
+ def matchRets1 = matchApi(esRets)
+ def matchRets2 = selectRestByCredit(matchRets1)
+ println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
+
+ def updateRets = processFieldsFromMatch(c , matchRets2)
+ println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
+ def precentMap = fieldUpdate(totalCount, updateRets)
+ println c +" "+precentMap.toString()
+ updateDB(c, updateRets)
+ println "------------------------------------------------------------------------------"
+ def pre1 = 0
+ def pre2 = 0
+ if(esRets){
+ pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
+ pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
+ }
+ analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2]
+ }
+ return analysRets;
+ }
+
+
+
+
+ def addFieldFromRonghe(c, totalCount){
+ sourceMap.each{seaweedmapSource,esWebSource->
+ def rhRets = findRongheDists(c, seaweedmapSource)
+ def esRets = findEsDistrict(c, esWebSource)
+ def updateRets = processFieldsFromRh(c, rhRets, esRets)
+
+ println "【通过融合关系填充字段】网站:"+seaweedmapSource +" 第三方融合表:"+rhRets.size() +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, rhRets.size()*100/totalCount)+"%"
+ def precentMap = fieldUpdate(totalCount, updateRets)
+ println c +" "+precentMap.toString()
+ updateDB(c, updateRets)
+ println "----------------------------------------------------------------------------------"
+ }
+ }
+
+ def processFieldsFromRh(c, rhRets, esRets){
+ def fields = fieldsMap.keySet() as List
+ def fieldValueMap = [:]
+ for(def rhDist : rhRets){
+ def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields)
+ if(!dist){
+ println c+" "+rhDist.commid+" 不存在"
+ continue
+ }
+ def esDbo = esRets.get(c + "#" + rhDist.id)
+ if(!esDbo){
+ println c+" "+rhDist.id+" 不存在[ES]"
+ continue
+ }
+ def newValues = [:]
+ fieldsMap.each{seaweedfield,esField->
+ def val = dist.get(seaweedfield)
+ if(!val){
+ def esVal = esDbo.get(esField)
+ if(esVal){
+ def newVal = null
+ if(seaweedfield == "estate_charge"){
+ newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
+ newVal = newVal.toString()
+ }
+ if(seaweedfield == "build_date"){
+ newVal = FiledStrand.processFinishBuildingDate(esVal)
+ }
+ if(newVal){
+ newValues.put(seaweedfield, newVal)
+ }
+ }
+ }
+ }
+ if(newValues){
+ fieldValueMap.put(dist._id.toString(), newValues)
+ }
+ }
+
+ println fieldValueMap.size()
+ return fieldValueMap;
+
+ }
+
+ def processFieldsFromMatch(c,matchRets){
+ def fields = fieldsMap.keySet() as List
+ fields << "city"
+ def fieldValueMap = [:]
+ for(def matchDist : matchRets){
+
+ def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
+ if(!dist){
+ println c+" "+matchDist.match_id+" 不存在"
+ continue
+ }
+
+ def newValues = [:]
+ fieldsMap.each{seaweedfield,esField->
+ def val = dist.get(seaweedfield)
+ if(!val){
+ def esVal = matchDist.get(esField)
+ if(esVal){
+ def newVal = null
+ if(seaweedfield == "estate_charge"){
+ newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
+ newVal = newVal.toString()
+ }
+ if(seaweedfield == "build_date"){
+ newVal = FiledStrand.processFinishBuildingDate(esVal)
+ }
+ if(seaweedfield == "estate_company" || seaweedfield == "developer"){
+ newVal = FiledStrand.processCompany(esVal)
+ }
+ if(seaweedfield =="virescence_percent"){
+ newVal = FiledStrand.processVirescencePercent(esVal)
+ }
+
+ if(seaweedfield =="cubage_percent"){
+ newVal = FiledStrand.processCubagePercent(esVal)
+ }
+
+ if(seaweedfield =="all_area" || seaweedfield =="build_area"){
+ newVal = FiledStrand.processArea(esVal)
+ newVal = newVal.toString()
+ }
+ if(seaweedfield =="household_count"){
+ newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
+ }
+
+ if(seaweedfield =="loudong_count"){
+ newVal = FiledStrand.processLoudongCount(esVal)
+ }
+
+ if(seaweedfield =="car_site_count"){
+ newVal = FiledStrand.processCarSite(esVal)
+ }
+ if(seaweedfield =="car_site"){
+ newVal = FiledStrand.processCarSiteDesc(esVal)
+ }
+
+ if(seaweedfield =="property_life"){
+ newVal = FiledStrand.processPropertyLife(esVal)
+ }
+
+ if(seaweedfield =="alias"){
+ newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
+ }
+
+ if(seaweedfield =="address"){
+ newVal = FiledStrand.processAddress(esVal, dist.city)
+ }
+
+ if(newVal){
+ newValues.put(seaweedfield, newVal)
+ }
+ }
+ }
+ }
+ if(newValues){
+ println dist._id.toString()+" "+newValues.toString()
+ fieldValueMap.put(dist._id.toString(), newValues)
+ }
+ }
+
+ return fieldValueMap;
+ }
+
+
+
+ def findRongheDists(def city, def source){
+ def keys = new BasicDBObject()
+ keys.put("commid", 1)
+ keys.put("city", 1)
+ keys.put("id", 1)
+ keys.put("source", 1)
+ def dbcursor = SEAWEEDMAP_COLL.find(new BasicDBObject().append("city", city).append("source", source), keys)
+ def rets = []
+ for(def d : dbcursor){
+ rets << ["commid":d.commid, "city":d.city, "source":d.source, "id":d.id]
+ }
+ return rets;
+ }
+
+ def findEsDistrict(c, esWebSource){
+ def map = [:]
+ ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
+ if(r.id && !map.containsKey(c+"#"+r.id)){
+ map.put(c+"#"+r.id, r)
+ }
+ })
+ return map
+ }
+
+
+ def findEsDatas(c, esWebSource){
+ def list = []
+ ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
+ list << r
+ })
+ return list
+ }
+
+ def fieldUpdate(total,updateRets){
+
+ def countMap = [:]
+
+ for(def rm : updateRets){
+ def valMap = rm.value
+ fieldsMap.each{seaweedfield,esField->
+ if(valMap.containsKey(seaweedfield)) {
+ if(countMap.containsKey(seaweedfield)) {
+ countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
+ }else{
+ countMap.put(seaweedfield, 1)
+ }
+ }
+ }
+ }
+
+ println countMap.toString()
+ def precentMap = [:]
+ fieldsMap.each() {key,skey->
+ if(countMap.containsKey(key)) {
+ precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
+ }else {
+ precentMap.put(skey, 0)
+ }
+ }
+ return precentMap;
+ }
+
+
+ def matchApi(esRets){
+ def rets = new CopyOnWriteArrayList()
+ def totalCount = new AtomicLong()
+ final CountDownLatch latch = new CountDownLatch(esRets.size())
+ for(def i=0;i=0.9){
+ matchRets1 << r
+ }
+ }
+ return matchRets1
+ }
+
+ def matching(data){
+ def name = data.name
+ if(data.name && data.name.contains("·")){
+ name = data.name.replace("·", "")
+ }
+
+ def flag = "名称匹配"
+ def dist = matchDist(data.city, data.region, name)
+
+ if(!dist){
+ dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district")
+ flag = "映射别名匹配"
+ }
+ if(!dist) {
+ def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district")
+ if(rets && rets.size()==1){
+ flag = "别名匹配"
+ dist = rets[0]
+ }
+ }
+
+ if(dist){
+ data.flag = flag
+ data.match_id = dist._id.toString()
+ data.match_region = dist.region
+ data.match_name = dist.name
+ data.credit = 0.95
+ }else{
+ def jsonObject = OpenApiClient.addressMatch(data.city, name, "property", false,data.region, null)
+ if(jsonObject!=null && jsonObject.precise==-1){
+ if(data.address && isStandardRoad(data.address)){
+ jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false,data.region, name)
+ }else{
+ data.flag = "匹配接口-地址"
+ }
+ }else{
+ data.flag = "匹配接口-名称"
+ }
+ if(jsonObject != null && jsonObject.precise!=-1) {
+ data.match_id = jsonObject.get("id")
+ data.match_region = jsonObject.get("district")
+ data.match_name = jsonObject.get("name")
+ data.credit = jsonObject.get("credit")
+ }
+ }
+ if(data.match_id){
+ return data
+ }
+ return null
+ }
+
+
+ def updateDB(c, updateRets){
+
+ for(def rm : updateRets) {
+ def id = rm.key
+ def valMap = rm.value
+
+ BasicDBObject udata = new BasicDBObject()
+ valMap.each {k,v->
+ udata.put(k,v)
+ }
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
+ }
+ }
+
+ def matchDist(city, region, name){
+ def dist = CommonDatautils.findDistrict(city, region, name, "district")
+ return dist;
+ }
+
+
+
+ def isStandardRoad(def s){
+ def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
+ def matcher = pattern.matcher(s)
+ if(matcher.find()){
+ return true;
+ }
+ return false
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy
new file mode 100644
index 000000000..945227e7a
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy
@@ -0,0 +1,71 @@
+package com.fangjia.script.gj.district
+
+import com.fangjia.core.RegionData
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.looplineinfo.InsidePolygon
+import com.fangjia.script.looplineinfo.Point
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+
+class SameDistName {
+
+ static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
+
+ static main(args) {
+ def task = new SameDistName()
+ task.run()
+ }
+
+ def run(){
+ def dists = CommonDatautils.findAllDistrictByCity("常州", "district", ["city","region", "name", "address","lat", "lng"])
+ def map = [:]
+ for(def d : dists){
+ def list = map.get(d.name)
+ if(!list){
+ list = []
+ map.put(d.name, list)
+ }
+ list << d
+ }
+
+ def datas = []
+ map.each {k,v->
+ if(v && v.size()>=2){
+ datas.addAll(v)
+ }
+ }
+
+ def retDats = []
+ retDats << ["城市","区域","名称","地址","区域状况"]
+ if(datas){
+ def regionPointsMap = findRegionRange("常州")
+ for(def d : datas){
+ def lng = d.lng/10000000000.0
+ def lat = d.lat/10000000000.0
+ def flag = isInArea(lng, lat, regionPointsMap.get(d.region))
+ retDats << [d.city, d.region, d.name, d.address, flag]
+ }
+ }
+ ExcelUtils.write("D:\\数据\\小区\\相同小区不同区域.xlsx", retDats, "数据")
+ }
+
+ def findRegionRange(city){
+ def q = new BasicDBObject()
+ q.put("city",city)
+ def dbcursor = POLYLINECOLL.find(q)
+ def regionMap = [:]
+ for(def d : dbcursor){
+ def region = RegionData.formatRegion(city,d.region)
+ regionMap.put(region, d.polyline)
+ }
+ return regionMap
+ }
+
+ def isInArea(lng, lat, points){
+ List pointList = InsidePolygon.createPoint2(points);
+ def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
+ return flag;
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy
index c133d7283..aa3c3c62c 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy
@@ -1,6 +1,8 @@
package com.fangjia.script.gj.es
+import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ElasticUtils
+import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
@@ -8,16 +10,39 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
-findLogs()
+//findLogs()
+//findFangjiaLogs()
+stat()
+def stat(){
+ def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1)
+ def map=[:]
+ for(def d : datas){
+ def count = map.get(d['C'])
+ if(!count){
+ map.put(d['C'], 1)
+ }else{
+ map.put(d['C'], count+1)
+ }
+ }
+
+ map.each {k,v->
+ println k +" "+v
+ }
+}
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+
+// def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
+// def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
+// QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
+
def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com")
- QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
+ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size()
def set = new HashSet()
@@ -39,4 +64,48 @@ def findLogs(){
}
return total
+}
+
+
+def findFangjiaLogs(){
+
+ def startDate = DateUtil.parseDate("20230329", "yyyyMMdd")
+ def endDate = DateUtil.parseDate("20230429", "yyyyMMdd")
+ Calendar calendar = Calendar.getInstance()
+ calendar.setTime(startDate)
+ def list = []
+ def data = [["mid", "ip", "访问时间"]]
+ while (startDate.before(endDate)) {
+ calendar.add(Calendar.DATE, 1)
+ def nextDate = calendar.getTime()
+
+// def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
+// def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
+ QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false);
+
+
+ def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
+ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
+ def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
+ println houses.size()
+
+ if (houses) {
+ for (def d : houses) {
+ def reqStr = d.request.toString()
+ def mid = null
+ if (reqStr.contains("mid=")) {
+
+ mid = reqStr.substring(reqStr.indexOf("mid=") + 4, reqStr.indexOf("mid=") + 8)
+ }
+ def time = d.get("@timestamp").toString()
+ time = time.substring(0, time.indexOf("T"))
+ def ip = d.remote_addr
+ data << [mid, ip, time]
+ }
+ }
+
+ startDate = nextDate
+ }
+
+ ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志")
}
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy
index 444fddbfc..ea7d46e2b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.es
+import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -9,14 +10,18 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
-findLogs()
-
+//findLogs()
+findFangjiaLogs()
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+ def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
+ def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
+
+ QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
def q = QueryBuilders.fuzzyQuery("request", "/jnBank")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder);
@@ -53,3 +58,35 @@ def findLogs(){
return total
}
+
+
+
+
+
+def findFangjiaLogs(){
+ def total = 0
+ try{
+
+// String sql = "select * from fangjia-log* where request like '/jnBank%'"
+// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+ def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
+ def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
+
+ QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
+ def q = QueryBuilders.fuzzyQuery("request", "/enquire")
+ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
+ def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder);
+ println houses.size()
+ def set = new HashSet()
+ if(houses) {
+ for (def d : houses) {
+
+ }
+ }
+ println set
+ }catch(Exception e){
+ e.printStackTrace()
+ }
+
+ return total
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy
new file mode 100644
index 000000000..ee95b0621
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaoDeMapSearchTest.groovy
@@ -0,0 +1,556 @@
+package com.fangjia.script.gj.location
+
+import com.fangjia.core.RegionData
+import com.fangjia.core.util.BaiDuApiUtils
+import com.fangjia.core.util.GeocodeUtils
+import com.fangjia.core.util.HttpRequest
+import com.fangjia.script.gj.OpenApiUtils
+import com.fangjia.script.gj.district.FangjiaClientUtils
+import com.fangjia.script.linxsh.util.AddressUtil
+import com.fangjia.script.util.CosineSimilarAlgorithm
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.OpenApiClient
+import net.sf.json.JSONObject
+
+import java.util.regex.Pattern
+
+class GaoDeMapSearchTest {
+
+ def baiduCityCodeMap = ["北京": "131", "天津": "332", "石家庄": "150", "唐山": "265", "秦皇岛": "148", "邯郸": "151", "邢台": "266", "保定": "307", "张家口": "264", "承德": "207", "沧州": "149", "廊坊": "191", "衡水": "208", "太原": "176", "大同": "355", "阳泉": "357", "长治": "356", "晋城": "290", "朔州": "237", "晋中": "238", "运城": "328", "忻州": "367", "临汾": "368", "吕梁": "327", "呼和浩特": "321", "包头": "229", "乌海": "123", "赤峰": "297", "通辽": "64", "鄂尔多斯": "283", "呼伦贝尔": "61", "巴彦淖尔": "169", "乌兰察布": "168", "兴安盟": "62", "锡林郭勒盟": "63", "阿拉善盟": "230", "沈阳": "58", "大连": "167", "鞍山": "320", "抚顺": "184", "本溪": "227", "丹东": "282", "锦州": "166", "营口": "281", "阜新": "59", "辽阳": "351", "盘锦": "228", "铁岭": "60", "朝阳": "280", "葫芦岛": "319", "长春": "53", "吉林市": "55", "四平": "56", "辽源": "183", "通化": "165", "白山": "57", "松原": "52", "白城": "51", "延边朝鲜族自治州": "54", "哈尔滨": "48", "齐齐哈尔": "41", "鸡西": "46", "鹤岗": "43", "双鸭山": "45", "大庆": "50", "伊春": "40", "佳木斯": "42", "七台河": "47", "牡丹江": "49", "黑河": "39", "绥化": "44", "大兴安岭地区": "38", "上海": "289", "南京": "315", "无锡": "317", "徐州": "316", "常州": "348", "苏州": "224", "南通": "161", "连云港": "347", "淮安": "162", "盐城": "223", "扬州": "346", "镇江": "160", "泰州": "276", "宿迁": "277", "杭州": "179", "宁波": "180", "温州": "178", "嘉兴": "334", "湖州": "294", "绍兴": "293", "金华": "333", "衢州": "243", "舟山": "245", "台州": "244", "丽水": "292", "合肥": "127", "芜湖": "129", "蚌埠": "126", "淮南": "250", "马鞍山": "358", "淮北": "253", "铜陵": "337", "安庆": "130", "黄山": "252", "滁州": "189", "阜阳": "128", "宿州": "370", "巢湖": "251", "六安": "298", "亳州": "188", "池州": "299", "宣城": "190", "福州": "300", "厦门": "194", "莆田": "195", "三明": "254", "泉州": "134", "漳州": "255", "南平": "133", "龙岩": "193", "宁德": "192", "南昌": "163", "景德镇": "225", "萍乡": "350", "九江": "349", "新余": "164", "鹰潭": "279", "赣州": "365", "吉安": "318", "宜春": "278", "抚州": "226", "上饶": "364", "济南": "288", "青岛": "236", "淄博": "354", "枣庄": "172", "东营": "174", "烟台": "326", "潍坊": "287", "济宁": "286", "泰安": "325", "威海": "175", "日照": "173", "莱芜": "124", "临沂": "234", "德州": "372", "聊城": "366", "滨州": "235", "菏泽": "353", "郑州": "268", "开封": "210", "洛阳": "153", "平顶山": "213", "安阳": "267", "鹤壁": "215", "新乡": "152", "焦作": "211", "濮阳": "209", "许昌": "155", "漯河": "344", "三门峡": "212", "南阳": "309", "商丘": "154", "信阳": "214", "周口": "308", "驻马店": "269", "武汉": "218", "黄石": "311", "十堰": "216", "宜昌": "270", "襄阳": "156", "鄂州": "122", "荆门": "217", "孝感": "310", "荆州": "157", "黄冈": "271", "咸宁": "362", "随州": "371", "恩施土家族苗族自治州": "373", "仙桃": "1713", "潜江": "1293", "天门": "2654", "神农架林区": "2734", "长沙": "158", "株洲": "222", "湘潭": "313", "衡阳": "159", "邵阳": "273", "岳阳": "220", "常德": "219", "张家界": "312", "益阳": "272", "郴州": "275", "永州": "314", "怀化": "363", "娄底": "221", "湘西土家族苗族自治州": "274", "广州": "257", "韶关": "137", "深圳": "340", "珠海": "140", "汕头": "303", "佛山": "138", "江门": "302", "湛江": "198", "茂名": "139", "肇庆": "338", "惠州": "301", "梅州": "141", "汕尾": "339", "河源": "200", "阳江": "199", "清远": "197", "东莞": "119", "中山": "187", "潮州": "201", "揭阳": "259", "云浮": "258", "南宁": "261", "柳州": "305", "桂林": "142", "梧州": "304", "北海": "295", "防城港": "204", "钦州": "145", "贵港": "341", "玉林": "361", "百色": "203", "贺州": "260", "河池": "143", "来宾": "202", "崇左": "144", "海口": "125", "三亚": "121", "五指山": "1644", "琼海": "2358", "儋州": "1215", "文昌": "2758", "万宁": "1216", "东方": "2634", "定安": "1214", "屯昌": "1641", "澄迈": "2757", "临高": "2033", "白沙黎族自治": "2359", "昌江黎族自治": "1642", "乐东黎族自治": "2032", "陵水黎族自治": "1643", "保亭黎族苗族自治": "1217", "琼中黎族苗族自治": "2031", "重庆": "132", "成都": "75", "自贡": "78", "攀枝花": "81", "泸州": "331", "德阳": "74", "绵阳": "240", "广元": "329", "遂宁": "330", "内江": "248", "乐山": "79", "南充": "291", "眉山": "77", "宜宾": "186", "广安": "241", "达州": "369", "雅安": "76", "巴中": "239", "资阳": "242", "阿坝藏族羌族自治州": "185", "甘孜藏族自治州": "73", "凉山彝族自治州": "80", "贵阳": "146", "六盘水": "147", "遵义": "262", "安顺": "263", "铜仁地区": "205", "黔西南布依族苗族自治州": "343", "毕节地区": "206", "黔东南苗族侗族自治州": "342", "黔南布依族苗族自治州": "306", "昆明": "104", "曲靖": "249", "玉溪": "106", "保山": "112", "昭通": "336", "丽江": "114", "临沧": "110", "楚雄彝族自治州": "105", "红河哈尼族彝族自治州": "107", "文山壮族苗族自治州": "177", "普洱": "108", "西双版纳傣族自治州": "109", "大理白族自治州": "111", "德宏傣族景颇族自治州": "116", "怒江傈僳族自治州": "113", "迪庆藏族自治州": "115", "拉萨": "100", "昌都地区": "99", "山南地区": "97", "日喀则地区": "102", "那曲地区": "101", "阿里地区": "103", "林芝地区": "98", "西安": "233", "铜川": "232", "宝鸡": "171", "咸阳": "323", "渭南": "170", "延安": "284", "汉中": "352", "榆林": "231", "安康": "324", "商洛": "285", "兰州": "36", "嘉峪关": "33", "金昌": "34", "白银": "35", "天水": "196", "武威": "118", "张掖": "117", "平凉": "359", "酒泉": "37", "庆阳": "135", "定西": "136", "陇南": "256", "临夏回族自治州": "182", "甘南藏族自治州": "247", "西宁": "66", "海东地区": "69", "海北藏族自治州": "67", "黄南藏族自治州": "70", "海南藏族自治州": "68", "果洛藏族自治州": "72", "玉树藏族自治州": "71", "海西蒙古族藏族自治州": "65", "银川": "360", "石嘴山": "335", "吴忠": "322", "固原": "246", "中卫": "181", "乌鲁木齐": "92", "克拉玛依": "95", "吐鲁番地区": "89", "哈密地区": "91", "昌吉回族自治州": "93", "博尔塔拉蒙古自治州": "88", "巴音郭楞蒙古自治州": "86", "阿克苏地区": "85", "克孜勒苏柯尔克孜自治州": "84", "喀什地区": "83", "和田地区": "82", "伊犁": "90", "塔城地区": "94", "阿勒泰地区": "96", "石河子": "770", "阿拉尔": "731", "图木舒克": "792", "五家渠": "789", "香港特别行政区": "2912", "澳门特别行政区": "2911"]
+
+
+ static main(args) {
+ def task = new GaoDeMapSearchTest();
+ def path = "D:\\数据\\光大\\光大测试-V2.xlsx"
+
+// task.structureV2(path)
+// task.match(path)
+// task.mapSearch(path)
+ task.baiduMapSearch(path)
+ task.checkTrue(path)
+ }
+
+ def checkTrue(path){
+ ExcelUtils.multiThreadingModify(path,{row->
+ if(!row['Z']){
+ if(row['Q']){
+ row['Z'] = "T"
+ }else{
+ if(row['Y']){
+ def address = ''
+ def poi = row['G']
+ if(row['E']){
+ address = address + row['E']
+ }
+ if(row['F']){
+ address = address + row['F']
+ }
+ if(address && isStandardRoad(address)){
+ def mapAddr = row['U']
+ if(mapAddr && mapAddr.contains(address)){
+ row['Z'] = "T"
+ }
+ }else if(poi){
+ def mapName = row['T']
+ if(mapName && mapName==poi){
+ row['Z'] = "T"
+ }
+ }
+ }
+ }
+ }
+ })
+ }
+
+
+ def isStandardRoad(def s){
+ def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
+ def matcher = pattern.matcher(s)
+ if(matcher.find()){
+ return true;
+ }
+ return false
+ }
+
+
+ def mapSearch(filePath){
+ ExcelUtils.multiThreadingModify(filePath,{row->
+ if(!row['M'] && !row['R']){
+ def city = row['C']
+ def region = row['D']
+ def address = ''
+ def poi = row['G']
+ if(row['E']){
+ address = address + row['E']
+ }
+ if(row['F']){
+ address = address + row['F']
+ }
+ def rets = []
+ if(address){
+ def ret1 = OpenApiUtils.getGaoDeSearchLocation(city, region, address)
+ rets.addAll(ret1)
+ }
+ if(!rets && poi){
+ def ret2 = OpenApiUtils.getGaoDeSearchLocation(city, region, poi)
+ rets.addAll(ret2)
+ }
+
+// if(!map && rets && rets.size()>0){
+// map = rets[0]
+// }
+
+ def bdRes = searchMapBySimilar(city, region, poi, address, rets);
+ if (bdRes) {
+ row['R'] = bdRes.uid
+ row['S'] = bdRes.region
+ row['T'] = bdRes.name
+ row['U'] = bdRes.address
+ row['V'] = bdRes.type
+ row['W'] = bdRes.score
+ if(bdRes.matchType){
+ row['X'] = bdRes.matchType
+ }
+ row['Y'] = "高德"
+ }
+ }
+ },0)
+ }
+
+
+
+ def findStandardObject(def city, def name, def rets) {
+ def map;
+ if (rets) {
+ if (rets.size() >= 1) {
+ //相似里找有精确地址
+ for (def mapObj : rets) {
+ mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
+ if (mapObj.address && findPattern(mapObj.address)) {
+ map = mapObj;
+ break;
+ }
+ }
+ if(!map){
+ //交带有交叉口地址
+ for (def mapObj : rets) {
+ if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) {
+ if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){
+ map = mapObj;
+ break;
+ }
+ }
+ }
+ }
+ }
+// if (!map) {
+// map = rets[0]
+// }
+ }
+ return map;
+ }
+
+
+ def structureV2(path){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ if(row['A']){
+ def jsonObj = FangjiaClientUtils.addressStructure(null, row['A'])
+ if(jsonObj && jsonObj.result){
+ if(jsonObj.result.province){
+ row['B'] = jsonObj.result.province
+ }
+ if(jsonObj.result.city){
+ def cityStr = null
+ if(jsonObj.result.city=="市辖区"){
+ cityStr = jsonObj.result.province
+ }else{
+ cityStr = jsonObj.result.city
+ }
+ row['C'] = RegionData.formatCity(cityStr)
+ }
+
+ if(jsonObj.result.district){
+ row['D'] = jsonObj.result.district
+ }
+
+ if(jsonObj.result.street){
+ row['E'] = jsonObj.result.street
+ }
+
+ if(jsonObj.result.streetNumber){
+ row['F'] = jsonObj.result.streetNumber
+ }
+
+ if(jsonObj.result.name){
+ row['G'] = jsonObj.result.name
+ }
+
+ if(jsonObj.result.buildingNumber){
+ row['H'] = jsonObj.result.buildingNumber
+ }
+
+ if(jsonObj.result.unitNumber){
+ row['I'] = jsonObj.result.unitNumber
+ }
+ if(jsonObj.result.roomNumber){
+ row['J'] = jsonObj.result.roomNumber
+ }
+
+ if(jsonObj.result.formatAddress){
+ row['K'] = jsonObj.result.formatAddress
+ }
+
+ if(jsonObj.result.addrVerify){
+ row['L'] = jsonObj.result.addrVerify
+ }
+ }
+ }
+ })
+ }
+
+
+
+ //可以根据测试数据添加 推荐楼盘名
+ def match(path){
+ ExcelUtils.modify(path, { row ->
+ def city = row['C']
+ if(city && !row['M']){
+ def address = ''
+ def suggestName = row['G']
+ if(row['E']){
+ address = address + row['E']
+ }
+ if(row['F']){
+ address = address + row['F']
+ }
+ def region = row['D']
+ if(!row['M'] && address){
+ city = RegionData.formatCity(city)
+ if(city){
+ region = RegionData.formatRegion(city, region)
+ }
+ println "[match] city:"+city+" region:"+region+" address:"+address
+ def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, suggestName)//"property"
+ Thread.sleep(2000)
+ if(jsonObject != null && jsonObject.precise!=-1) {
+ row['M'] = jsonObject.get("city")
+ row['N'] = jsonObject.get("district")
+ row['O'] = jsonObject.get("name")
+ row['P'] = jsonObject.get("propertyType")
+ row['Q'] = jsonObject.get("id")
+ }
+ }
+ }
+ })
+ }
+
+ def baiduMapSearch(path){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ if(row['C'] && !row['M'] && !row['R']){
+ def city = row['C']
+ def region = row['D']
+ def address = ''
+ def poi = row['G']
+ if(row['E']){
+ address = address + row['E']
+ }
+ if(row['F']){
+ address = address + row['F']
+ }
+ def baiduCode = getBaiduMapCode(city);
+ if(baiduCode){
+ def rets = null
+ if(address && row['G']){
+ rets = getBiduSearchDatas(baiduCode , address, row['G'], null)
+ }else if(row['G']){
+ rets = getBiduSearchDatas(baiduCode , null, row['G'], rets)
+ }else if(address){
+ rets = getBiduSearchDatas(baiduCode , null, address, rets)
+ }
+
+ def bdRes = searchMapBySimilar(city, region, poi, address, rets);
+ if (bdRes) {
+ row['R'] = bdRes.uid
+ row['S'] = bdRes.region
+ row['T'] = bdRes.name
+ row['U'] = bdRes.address
+ row['V'] = bdRes.type
+ row['W'] = bdRes.score
+ if(bdRes.matchType){
+ row['X'] = bdRes.matchType
+ }
+ row['Y'] = "百度"
+ }
+ println row['B'] +" "+ bdRes
+ }
+ }
+ });
+ }
+
+
+ def getSimpleAddress(city, address){
+ def jsonObj = FangjiaClientUtils.addressStructure(city, address)
+ def old_addr = ""
+ def map = [:]
+ if(jsonObj && jsonObj.result){
+
+ if(jsonObj.result.province){
+ map.pro = jsonObj.result.province
+ }
+
+ if(jsonObj.result.city){
+ if(jsonObj.result.city=="市辖区"){
+ map.city = jsonObj.result.province
+ }else{
+ map.city = jsonObj.result.city
+ }
+ map.city = RegionData.formatCity(map.city)
+ }
+
+ if(jsonObj.result.district){
+ map.region = jsonObj.result.district
+ }
+ if(jsonObj.result.street){
+ def street = jsonObj.result.street
+ if(street){
+ old_addr = street
+ map.street = street.replace("交叉口","").replace("交叉","")
+ }
+ }
+ if(jsonObj.result.streetNumber){
+ def street_number = jsonObj.result.streetNumber
+ if(street_number){
+ old_addr = old_addr + street_number
+ }
+ }
+ }
+ map.addr = old_addr.replace("交叉口","").replace("交叉","")
+ return map;
+ }
+
+ def getBaiduMapCode(city){
+ if(city){
+ return baiduCityCodeMap[city]
+ }
+ return city
+ }
+
+ def getBiduSearchDatas(baiduCode , simpleAddr, nameKey, rets){
+ def searchKey = ""
+ if(simpleAddr){
+ searchKey = simpleAddr +" "+nameKey
+ }else{
+ searchKey = nameKey
+ }
+ def datas = searchBaiDuMap(searchKey, baiduCode.toInteger());
+ if(!rets){
+ return datas;
+ }else{
+ def ids = []
+ for(def r : rets){
+ if(!ids.contains(r.uid)){
+ ids << r.uid
+ }
+ }
+ for(def d : datas){
+ if(!ids.contains(d.uid)){
+ rets << d
+ }
+ }
+ return rets;
+ }
+ return null;
+ }
+
+
+ static def searchBaiDuMap(String address, int cityCode) {
+ def rets = []
+ try {
+ Thread.sleep(500)
+ // String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&&auth=Dae1RaNR1LK8xdJTxN8SQOTRa2AH%3DL7vuxLNVENTxRBt6BG6B6WJWwGvzBU%40yYxvkGcuVtvvhguVtvyheuVtcvY1SGpuLtB2QKAZzvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwBVDzvFosSE21wiKDv7uvhgMuzVVtvrMhuENLtSrrlrIKHf0wd0vyMO7FOUySy&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a";
+ String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=after_baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=19&from=webmap&sug_forward=&auth=E8LCawOwRbEOx6JVTvCIBI6W7zSz5vSXuxLNVENTRVLtUDHUDUYKYyHxzBU%40yYxvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwDWBWvveh3uxtwiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3VP8JSrrlrIKHf0wd0vyMO7FOUS7F&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a&device_ratio=2&tn=B_NORMAL_MAP&nn=0&u_loc=13513133,3631861&ie=utf-8&t=${System.currentTimeMillis()}&newfrom=zhuzhan_webmap"
+ def content = new HttpRequest().get(url, null, "utf-8")
+ JSONObject json = JSONObject.fromObject(content)
+ if (json && json.content && json.content.size() > 0) {
+ for (def loc : json.content) {
+// String name = formatName(address)
+ def x = loc.x
+ def y = loc.y
+ if(x){
+ x = x/100
+ }
+ if(y){
+ y = y/100
+ }
+ if(x && y){
+ def locJson = BaiDuApiUtils.geoconv(x, y, 6)
+ if(locJson){
+ def baiduCity = RegionData.parseCityInAddress(loc.area_name)
+ def bean = [:]
+// def lng2 = locJson.get("x")
+// def lat2 = locJson.get("y")
+// def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
+// def lng = gmap.get("lng")
+// def lat = gmap.get("lat")
+ bean.uid = loc.uid
+// bean.baidu_lng2 = lng2
+// bean.baidu_lat2 = lat2
+// bean.longitude = lng
+// bean.latitude = lat
+ bean.name = loc.name
+// if(loc.address_norm){
+// bean.address = getAddressInfo(loc.address_norm, loc.addr);
+// }
+ bean.address = loc.addr
+
+ if(!baiduCity){
+ baiduCity = RegionData.formatCity(loc.city_name)
+ }
+ bean.cityname = baiduCity
+ if(loc.city_name){
+ def regionStr = loc.area_name.replace(loc.city_name, "")
+ def baiduRegion = RegionData.formatRegion(baiduCity, regionStr);
+ bean.region = baiduRegion
+ }
+
+ if(loc.std_tag){
+ bean.type = loc.std_tag
+ }
+ bean.flag = loc.acc_flag
+ rets << bean;
+ }
+ }
+ }
+ }
+ } catch(Exception e) {
+ searchBaiDuMap(address, cityCode)
+ }
+ return rets;
+ }
+
+
+ def searchMapBySimilar(def city, def region, def newKey, def inpAddr, def rets){
+ def bo = null;
+ if(rets){
+ def maxScore = 0;
+ if(!bo || !maxScore){
+ for(def o :rets){
+ def addr = o.address
+ def mapKey = o.name
+ if(mapKey){
+ def score = 0
+ def nameScore = 0
+ if(newKey){
+ nameScore = CosineSimilarAlgorithm.getSimilarityRatio(mapKey, newKey);
+ }
+
+ def addr_score = 0
+ if(inpAddr) {
+ def baidu_addr_map = getSimpleAddress(o.cityname, o.address)
+ if(!baidu_addr_map.addr){
+ baidu_addr_map.addr = AddressUtil.formatAddress(o.cityname, o.address)
+ }
+// def baidu_addr_map = AddressUtil.formatAddress(o.cityname, o.address)
+ if(!baidu_addr_map.addr){
+ if(o.region && !region.contains(o.region)){
+ continue;
+ }
+ if(city!=o.cityname){
+ continue;
+ }
+ }
+ if (baidu_addr_map.addr) {
+ if(baidu_addr_map.addr.contains("与")){
+ def arrs = baidu_addr_map.addr.split("与")
+ for(def s1 : arrs){
+ def temp_addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, s1);
+ if(temp_addr_score > addr_score && (inpAddr.contains(baidu_addr_map.addr) || baidu_addr_map.addr.contains(inpAddr))){
+ addr_score = temp_addr_score
+ }
+ }
+ }else{
+ addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr_map.addr);
+ if(addr_score>0 && inpAddr && baidu_addr_map.street && !inpAddr.contains(baidu_addr_map.street) && !baidu_addr_map.street.contains(inpAddr)){
+ addr_score = 0
+ }
+ }
+
+ }
+ }
+
+ if(addr_score>0 && score==0){
+ addr_score = addr_score/2
+ }
+ def total_score = nameScore + addr_score
+// if(city==o.cityname){
+ if(total_score > maxScore){
+ maxScore = total_score;
+ o.city = o.cityname
+ bo = o
+ bo.score = maxScore
+
+ def list = []
+ if(addr_score>0){
+ list << "地址有关"
+ }
+ if(nameScore>0){
+ list << "名称相似"
+ }
+ if(list){
+ o.matchType = list.join(",")
+ }
+ }
+// if(bo && score!=1){
+// if(inpAddr){
+// def baidu_addr = getSimpleAddress(o.cityname, o.old_addr)
+// if(baidu_addr){
+// def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr);
+// //if(addr_score==1){
+// // o.matchType = "城市相同,名称相似,地址有关"
+// bo.score = bo.score + addr_score
+// maxScore = bo.score;
+// //}
+// }
+// }
+
+// for(def word : filters){
+// if(mapKeyName.toString().endsWith(word)){
+// mapKeyName = mapKeyName.replace(word, "")
+// }
+// if(newName.toString().endsWith(word)){
+// newName = newName.replace(word, "");
+// }
+// }
+//
+//
+// if(mapKeyName && newName && mapKeyName.length()>=2){
+// score = CosineSimilarAlgorithm.getSimilarityRatio(mapKeyName, newName);
+// if(score > maxScore){
+// maxScore = score
+// o.matchType = "名称相似"
+// o.city = obj.city
+// bo = o
+// bo.score = maxScore
+// }else{
+// if(mapKeyName.length() == newName.length()){
+// def mapKeyNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(mapKeyName, " ")
+// def newNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
+// if(mapKeyNameBY == newNameBY){
+// maxScore = 0.977;
+// o.matchType = "拼音相似"
+// o.city = obj.city
+// bo = o
+// bo.score = maxScore
+// }
+// }
+// }
+// }
+// }
+// }
+ }
+ }
+ }
+ }
+ return bo;
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
index f641c214c..2aae0edb5 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
@@ -32,7 +32,7 @@ public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
- def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
+ def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
static main(args) {
def task = new BeiKeCrawlByChrome();
@@ -44,7 +44,7 @@ public class BeiKeCrawlByChrome {
def run(){
- System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
+ System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
@@ -112,18 +112,20 @@ public class BeiKeCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def m : citys){
- driver.get(m.url+"/chengjiao/")
+ def city = m[0]
+ def url = m[1]
+ driver.get(url+"/chengjiao/")
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
- def href = m.url+ r.attr("href")
+ def href = url+ r.attr("href")
- println "开始抓取【${m.city}】【${regionText}】【${href}】"
- def detailUrls = getPageDeail(driver, m.city, href, regionText)
- println m.url +" 详情链接:"+detailUrls.size()
+ println "开始抓取【${city}】【${regionText}】【${href}】"
+ def detailUrls = getPageDeail(driver, city, href, regionText)
+ println url +" 详情链接:"+detailUrls.size()
}
}
@@ -183,7 +185,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
- def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
+ def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, region)
@@ -398,16 +400,13 @@ public class BeiKeCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
def list = []
+ def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"]
if(cityEles){
for(def e : cityEles){
- def href = e.attr("href")
-// if(e.text().contains("福州")){
- def cityMap = [:]
- cityMap.city = e.text()
- cityMap.url = "https:"+href
- list << cityMap
-// cityMap.put(e.text(), "https:"+href)
-// }
+ def href = "https:"+e.attr("href")
+ if (citys.contains(e.text())){
+ list << [e.text(), href]
+ }
}
}
return list
@@ -432,6 +431,7 @@ public class BeiKeCrawlByChrome {
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
+ Thread.sleep(10000)
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
index 3c869e61b..343a4a3a4 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
@@ -31,7 +31,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
- def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
+ def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -117,10 +117,11 @@ public class LianjiaCrawlByChrome {
}
def run(){
- System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
+ System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
+// cityMap = cityMap.reverse()
getLogin(driver)
crawlData(driver, cityMap)
@@ -184,12 +185,15 @@ public class LianjiaCrawlByChrome {
}
}
- def crawlData(driver, cityMap){
+ def crawlData(driver, cityList){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
- cityMap.each{city, cityUrl->
+ for(def ret : cityList){
+ def city = ret[0]
+ def cityUrl = ret[1]
//def detailUrls = getPageUrls(driver, cityUrl)
driver.get(cityUrl+"chengjiao/")
+ Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
@@ -251,7 +255,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
- def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
+ def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
@@ -473,17 +477,17 @@ public class LianjiaCrawlByChrome {
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
- def cityMap = [:]
- def citys= ["苏州"]
+ def cityList = []
+ def citys= ["昆山","九江","郑州","廊坊","漳州","咸阳","长春","重庆"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
- cityMap.put(e.text(), href)
+ cityList << [e.text(), href]
}
}
}
- return cityMap
+ return cityList
}
def getLogin(driver){
@@ -505,6 +509,7 @@ public class LianjiaCrawlByChrome {
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
+ Thread.sleep(20000)
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy
index b994fde09..f419acf6d 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy
@@ -21,10 +21,10 @@ class AddPriceTest {
task.run();
}
def run(){
- def filePath = "D:\\数据\\招商\\招商.xlsx"
-// addGXDPrice(filePath)
+ def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx"
+ addGXDPrice(filePath)
// addCcbPrice(filePath)
- addThirdPrice(filePath)
+// addThirdPrice(filePath)
}
@@ -104,12 +104,13 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
- if (d['M']) {
- def dist = CommonDatautils.findDistrictById(d['M'])
- if(dist){
- def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
- d['AO'] = gxdPrice
- }
+ if (d['B']) {
+// def dist = CommonDatautils.findDistrictById(d['B'])
+// if(dist){
+// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
+ def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
+ d['G'] = gxdPrice
+// }
}
})
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy
index 1bdbfcdc7..07ab82cce 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/ChongQingBlankTest.groovy
@@ -23,11 +23,16 @@ class ChongQingBlankTest {
static main(args) {
def task = new ChongQingBlankTest()
task.run()
+
}
def run(){
- ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row->
+ allRegionMap.each{k,v->
+ println k+" "+v.size()
+ }
+
+ /* ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row->
if(row['B'].toString()=="重庆市九龙坡区建工三村7号"){
def city = CommonUtils.parseCityInAddress(row["B"])
if(!city){
@@ -197,7 +202,7 @@ class ChongQingBlankTest {
}
}
}
- }, 0)
+ }, 0)*/
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
index a254cbdd7..175b2ebd1 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
@@ -27,8 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
- def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
-
+ def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产"]
static main(args) {
runBatch(new SynchDistrictPrice(), args);
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
index 5f9a10997..bdd887539 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
- def count1 = findTotalCount(source, 202301);
- def count2 = findTotalCount(source, 202302);
+ def count1 = findTotalCount(source, 202302);
+ def count2 = findTotalCount(source, 202303);
println source +" "+count1 +" "+count2
}
}