From 7bddc42298adb6a84e6311e3f089f91bd2d81460 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Jan 2022 12:20:33 +0800 Subject: [PATCH] update --- .../ccbevaluation/ExportOracleData.groovy | 6 +- .../ccbevaluation/data/ExcelDataTest.groovy | 127 ++++++ .../ccbevaluation/data/MatchMapData.groovy | 60 ++- .../ccbevaluation/ronghe/SSHHelper.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 4 +- .../script/gj/level/DistrictIndexBatch.groovy | 420 ++++++++++++++++++ .../script/thirdPriceCrawl/Esf58Crawl.groovy | 65 ++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 3 +- 8 files changed, 657 insertions(+), 30 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e86a065cc..24dbf7876 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -76,7 +76,7 @@ class ExportOracleData { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" } @@ -503,7 +503,7 @@ class ExportOracleData { allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" } @@ -522,7 +522,7 @@ class ExportOracleData { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy new file mode 100644 index 000000000..8373b928c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -0,0 +1,127 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ExcelUtils + +//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1) +//def cityDataMap = [:] +//for(def row : cityDatas){ +// if(row['D']){ +// cityDataMap.put(row['A']+"##"+row['B']+"##"+row['C'], row) +// } +//} +// +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据_1529.xlsx",{d-> +// +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// def key = d['A']+"##"+newRegion+"##"+d['C'] +// println key +// def datas = cityDataMap.get(key) +// if(datas){ +// d['F'] = datas['D'] +// d['G'] = datas['E'] +// d['H'] = datas['F'] +// d['I'] = datas['G'] +// d['J'] = datas['H'] +// +// } +//},0) + +def city = "广州" +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +// def newRegion = RegionData.formatRegion(city, d['A']) +// if(newRegion){ +// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null); +// if(rMap){ +// d['C'] = rMap.region +// d['D'] = rMap.name +// } +// } +// +//},0) + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",{d-> +// def newRegion = RegionData.formatRegion(city, d['E']) +// if(newRegion){ +// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['F'], null); +// if(rMap){ +// d['L'] = rMap.region +// d['M'] = rMap.name +// } +// } +//},0) + +//def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",0, 1) +//def idMaps = [:] +//for(def d : datas){ +// if(d['L']){ +// idMaps.put(d['L']+"##"+d['M'], d) +// } +//} +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +// if(d['C']){ +// def key = d['C']+"##"+d['D'] +// def rows = idMaps.get(key) +// if(rows){ +// d['E'] = rows['D'] +// d['F'] = rows['E'] +// d['G'] = rows['F'] +// } +// } +//},0) + + +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district") +// if(obj){ +// def year = CommonUtils.getDistrictAge(obj) +// if(year){ +// d['H'] = year +// } +// if(obj.alias){ +// d['K'] = obj.alias.join(" ") +// } +// +// } +//},0) + +def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1) +def scoreMap = [:] +for(def d : data1s){ + scoreMap.put(d['B'], d) +} + +def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1) +def levelMap = [:] +for(def d : data2s){ + levelMap.put(d['E'], d) +} + +ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> + if(d['E']){ + def row = scoreMap.get(d['E']) + if(row){ + d['I'] = row['G'] + d['J'] = row['H'] + d['K'] = row['I'] + d['L'] = row['J'] + d['M'] = row['K'] + d['N'] = row['L'] + d['O'] = row['M'] + } + + def data = levelMap.get(d['E']) + if(data){ + d['P'] = data['G'] + d['Q'] = data['H'] + d['R'] = data['I'] + d['S'] = data['J'] + d['T'] = data['K'] + d['U'] = data['L'] + d['V'] = data['M'] + } + } +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy index 51ea1deba..5e58e7b38 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -19,6 +19,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders +import java.util.concurrent.CopyOnWriteArrayList + class MatchMapData { def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol(); @@ -28,12 +30,12 @@ class MatchMapData { static main(args) { def task = new MatchMapData() -// task.run() + task.run() // task.run1() // task.run2() // task.run3() // task.exportHouse() - task.addDistrictLevel() +// task.addDistrictLevel() } @@ -135,16 +137,22 @@ class MatchMapData { } def run(){ - //"合肥", - def files = ["上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"] + //"广州", + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def files = ["合肥","广州","上海","北京","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"] for(def f : files){ def city = f.replace("-2","") def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365) def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear)) def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend) - def datas = [] - datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型"] + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def priceDatas = addPriceData(dists, priceMap); + def levelMap = calcuteLevel(priceDatas); + + def datas = new CopyOnWriteArrayList() + datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型","评级","均价"] ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d-> def regionCode = null if(d['E']){ @@ -192,13 +200,23 @@ class MatchMapData { d['N'] = prc.toInteger() } d['O'] = dist.estate_type2 + + + def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name) + if(dist.name == "绿苑公寓"){ + println dbo + } + if(dbo){ + d['P'] = dbo.level + d['Q'] = dbo.avg_price + } } }else{ d['M'] = "区域无法标准化" } } - datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O']] + datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O'], d['P'], d['Q']] },0) @@ -385,12 +403,12 @@ class MatchMapData { } def addDistrictLevel(){ +// def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"] def dataMap = null def weekend = DateUtils.getFirstDayOfWeek(new Date()); - for(def fn : files){ def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"]) def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend) @@ -399,15 +417,28 @@ class MatchMapData { dataMap = getDataMap(fn); - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\评级10级\\"+fn+".xlsx",{d-> def region = dataMap.get(d['C']) if(region){ def newRegion = RegionData.formatRegion(fn, region) if(newRegion){ - def level = levelMap.get( fn+"##"+newRegion+"##"+d['D']) - if(level){ - d['J'] = level + def dist = CommonDatautils.findDistrict(fn, newRegion, d['D'],"district") + if(!dist){ + dist = CommonDatautils.findSeaweedByMappingAlias(fn, newRegion, d['D'],"district") + if(!dist){ + def rets = CommonDatautils.findDistrictByAlias(fn, newRegion, d['D'],"district") + if(rets){ + dist = rets[0] + } + } + } + if(dist){ + def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name) + if(dbo){ + d['J'] = dbo.level + d['K'] = dbo.avg_price + } } } } @@ -472,7 +503,10 @@ class MatchMapData { } def dataMap = [:] for(def d : datas){ - dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level) + if(d.name=="绿苑公寓"){ + println d + } + dataMap.put(d.city+"##"+d.region+"##"+d.name, d) } return dataMap; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy index af4571228..705a5c1f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy @@ -105,7 +105,7 @@ class SSHHelper { public static void main(String[] args) { SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20211210")); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110")); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 70c90b65f..23945eac2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -705,7 +705,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","长隆小区") +// q.put("name","吴中路9弄小区") DBObject keys = new BasicDBObject(); if(fields){ @@ -829,6 +829,8 @@ class CommonDatautils { keys.put("lat", 1); keys.put("lng", 1); keys.put("name_cat", 1); + keys.put("name_cat", 1); + keys.put("name_cat", 1); DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy new file mode 100644 index 000000000..e901f862b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -0,0 +1,420 @@ +package com.fangjia.script.gj.level + +import com.fangjia.core.util.HttpRequest +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.JacksonUtil +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.apache.commons.httpclient.util.DateUtil +import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.zip.GZIPInputStream + +class DistrictIndexBatch extends BaseBatch{ + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc" + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + runBatch(new DistrictIndexBatch(), args); + } + + @Override + public void process(Object args) { + run(); + } + + def run(){ + def citys = ["上海"]//CommonDatautils.findAllCity(); + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) + calcute(city, dists) + } + } + + def calcute(city, dists){ + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def costDatas = getCostByCity(city) + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def datas = new CopyOnWriteArrayList() + datas << ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] + for(def i=0;i list = DbUtils.findAroundDistrict(d) + def avg_charge = null + def rank = null + if (list) { + list = sortByDistance(d, list); + avg_charge = gtAvgEstateCharge(list) + rank = getPriceInRegionRank(d, list, priceMap) + } + def total_price = getHousePrice(d, dists, priceMap); + def avg_cost = getAvgCost(d, costDatas) + + def total_price_index = 0 + if (total_price) { + total_price_index = 286.0134196 * (total_price - 531.2615913) / 313.3693508 + } + def avg_charge_index = 0 + if (avg_charge) { + avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241 + } + + def rank_index = 0 + if (rank) { + rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636 + } + + def avg_cost_index = 0 + if (avg_cost) { + avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959 + } + def potentiality = avg_cost_index + 4099.978248 + potentiality = DataHandleUtils.getDoubleRound(2, potentiality) + def map = [:] + map.id = new ObjectId().toString() + map.city = d.city + map.region = d.region + map.name = d.name + map.avg_total_price = total_price + map.avg_estate_charge = avg_charge + map.price_rank = rank + map.avg_cost = avg_cost + map.comp_value = potentiality + map.month = "2022-01" + map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") + + datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] + + def newUrl = targetUrl +"/"+map.id + httpSaveData(newUrl, map) + println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown(); + } + } + }) + } + latch.await(); + // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") + } + + def getHousePrice(d, dists, priceMap){ + def areas = findHouse(d, 20200101, 20220101) + def avg_area = null + if(areas){ + avg_area = areas.sum()/areas.size(); + }else{ + if(d.finish_building_date){ + def year = CommonUtils.getBuildYear(d.finish_building_date) + if(year){ + def startYear = (year.toString().substring(0,3)+"0").toInteger() + def endYear = startYear + 10 + + def selectDists = [] + for(def ad : dists){ + if(d.region == ad.region){ + def adYear = CommonUtils.getBuildYear(ad.finish_building_date) + if(adYear){ + adYear = adYear.toInteger() + if(adYear>=startYear && adYear<=endYear){ + selectDists << ad + } + } + } + } + + if(selectDists){ + def allAreas = [] + for(def sd : selectDists){ + def areaDatas = findHouse(sd, 20190101, 20220101) + allAreas.addAll(areaDatas) + } + avg_area = allAreas.sum()/allAreas.size(); + } + } + } + } + + if(avg_area){ + def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) + def total_price = (avg_price * avg_area)/10000.0 + def total = DataHandleUtils.getDoubleRound(2, total_price) + return total + } + return null + } + + + def gtAvgEstateCharge(list){ + def rets = [] + for(def i=0;i=3){ + break; + } + } + } + def avg_charge = rets.sum()/rets.size(); + avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge) + return avg_charge; + } + + def getPriceInRegionRank(d, nbLists, priceMap){ + if(nbLists){ + if(nbLists.size()==1){ + return 0.5; + }else{ + def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) + def count = 0 + if(avg_price){ + for(def nb : nbLists){ + def price = priceMap.get(nb.city+"##"+nb.region+"##"+nb.name) + if(price > avg_price){ + count ++ + } + } + } + def pre = (nbLists.size() - count)/nbLists.size() + pre = DataHandleUtils.getDoubleRound(2, pre) + return pre; + } + } + return null; + } + + def getCostByCity(city){ + def datas = queryPoi(city) + return datas + } + + def sortByDistance(d, list){ + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + } + //排序 + Collections.sort(list, new Comparator() { + @Override + int compare(Object o1, Object o2) {return o1.distance-o2.distance;} + }) + return list; + } + + def getAvgCost(d, costDatas){ + if(costDatas){ + def costs = [] + for(def c : costDatas){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, c.lat, c.lng) + if(distance <= 1000){ + costs << c.cost + } + } + if(costs){ + def avg_cost = costs.sum()/costs.size() + avg_cost = DataHandleUtils.getDoubleRound(2, avg_cost) + return avg_cost + } + + } + return null + } + + + def queryPoi(def city){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode,business.keytag as keytag,business.cost as cost FROM amap_poi_detail WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "市'") + sqlBuffer.append(" and business.cost>20 and business.cost<500") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def datas = [] + def tags = ["法国菜","东南亚菜","日本料理","火锅","海鲜","自助餐","西餐","浙菜","茶餐厅","酒吧","烤肉","潮州菜","云贵菜","韩国料理","中餐","特色火锅","西北菜","串串香","农家乐","烧烤","茶座","上海菜","意大利菜","素菜馆","蛋糕店","东北菜","湘菜","粤菜","咖啡厅","轰趴馆","清真菜","北京菜","川菜","餐饮服务,餐饮相关场所,餐饮相关"] + def index = 0 + while (rs.next()) { + index++ + def tag = rs.getString("keytag") + if(tags.contains(tag)){ + def dMap = [:] + dMap.cost = rs.getInt("cost") + def location = rs.getObject("location") + if(location){ + dMap.lng = location.x + dMap.lat = location.y + } + datas << dMap + }else{ + println tag + } + } + println index + return datas; + } + + + def findHouse(def dist, def startDate, def endDate){ + def nameQuery = getAllName(dist); + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT area FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+dist.city+"'") + sqlBuffer.append(" and region = '"+dist.region+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + sqlBuffer.append(" and (" + nameQuery + ")"); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def areas = [] + while (rs.next()){ + areas << rs.getDouble("area") + } + return areas; + } + + def getAllName(dist){ + Set aliasNames = new LinkedHashSet(); + aliasNames.add(dist.name); + if (dist != null && dist.alias != null) { + aliasNames.addAll(dist.alias); + } + + List namePars = new ArrayList(); + if(aliasNames!=null && aliasNames.size()>0) { + for(String name : aliasNames){ + if(name && name.trim().length() > 1){ + namePars.add("district_name ='" + name + "'"); + } + } + } + + if(namePars.size() > 0){ + return namePars.join(" OR ") + } + return null; + } + + + +// def updateHouseMatchDistrictId(def id, def match_id){ +// def header = [:] +// header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() +// String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" +// def body = """ +// { +// "script": { +// "source": "ctx._source.match_seaweed_id='${match_id}'", +// "lang": "painless" +// }, +// "query": { +// "term": { +// "_id": "${id}" +// } +// } +// } +// """ +// HttpRequest request = new HttpRequest() +// String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") +// println s +// } + + + + def httpSaveData(def newUrl,def params){ + def header = [:] + header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=") + doPost(newUrl,header, params, true, "utf-8") + } + + + def doPost(def targetUrl, def headers, def params, def isJson, def charset){ + try{ + URL url = new URL(targetUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setUseCaches(false); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + conn.setRequestProperty("User-Agent", UserAgents.get()); + String param = null; + BufferedWriter mWriter = null; + InputStreamReader inStream = null; + StringBuilder buffer = new StringBuilder(); + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + conn.setRequestProperty(head, headers.get(head)); + } + } else { + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" ); + } + + if (isJson) { + conn.setRequestProperty("Content-Type", "application/json; charset=" + charset); + param = JacksonUtil.beanToJson(params) + } + + conn.connect(); + + //println param.toString() + //输出流 + if (StringUtils.isNotEmpty(param)) { + mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset)); + mWriter.write(param); + mWriter.flush(); + mWriter.close(); + } + if (conn.getHeaderField("Content-Encoding") != null + && conn.getHeaderField("Content-Encoding").equals("gzip")) { + inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset); + } else { + inStream = new InputStreamReader(conn.getInputStream(), charset); + } + char[] bf = new char[1024]; + int len; + while ((len = inStream.read(bf)) != -1) { + buffer.append(bf, 0, len); + } + } catch (Exception e) { + e.printStackTrace() + } + return null; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index d369ef53a..ae2639418 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -38,9 +38,10 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", -// "上海":"sh|2", "北京":"bj|1","天津":"tj|18", +// "青岛":"qd|122", // "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", - def cityList = ["青岛":"qd|122","南京":"nj|172", +// "上海":"sh|2", + def cityList = [ "北京":"bj|1","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", @@ -261,7 +262,7 @@ class Esf58Crawl extends BaseBatch{ def newUrl = url+"?sort=param62_asc&key="+URLEncoder.encode(keyword, "utf-8") def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET); if(document){ - Elements elements = JQueryUtils.select([".list-cell a"], document) + Elements elements = JQueryUtils.select([".list-cell .li-row"], document) if(elements){ addSearchContent(city, url, keyword); }else{ @@ -279,9 +280,11 @@ class Esf58Crawl extends BaseBatch{ def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET); println "[${totalCount.get()}]关键字【${keyword}】 页:"+ newUrl if(document){ - Elements elements = JQueryUtils.select([".list-cell a"], document) - if(elements){ - saveDetailContent(city, document); + Elements infoElements = JQueryUtils.select([".list-cell .li-info"], document) + Elements priceElements = JQueryUtils.select([".list-cell .li-side"], document) + if(infoElements){ +// saveDetailContent(city, document); + saveLists(city, infoElements, priceElements) }else{ flag = false; } @@ -289,6 +292,46 @@ class Esf58Crawl extends BaseBatch{ flag = false; } page++; + if(page>50){ + flag = false; + } + } + } + + + def saveLists(city, infoElements, priceElements){ + for(def i=0;i") } - def avg_price = JQueryUtils.selectText(".price", detailDocument); + def avg_price = JQueryUtils.selectText(".average", detailDocument); def region; - def addr = JQueryUtils.selectText("td:contains(详细地址)+td", detailDocument); + def addr = JQueryUtils.selectText(".column-1:contains(小区地址) .value", detailDocument); if(info){ if(info.size()>=4){ - city = info[2].replace("小区","").trim() - region = info[3].replace("小区","").trim() + city = info[1].replace("小区","").trim() + region = info[2].replace("小区","").trim() } } if(!addr){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index c106ca013..3993f6663 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,8 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] - def sourceMap = ["fang.com":"搜房","cityhouse.cn":"城市房产","5i5j.com":"我爱我家"] + def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多", + "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run()