update
This commit is contained in:
+33
-33
@@ -60,39 +60,39 @@ class BigdataDealBatch extends BaseBatch{
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// for(def city : citys){
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成"
|
||||
|
||||
|
||||
}
|
||||
|
||||
+6
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220518, 20220619);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220618, 20220719);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
@@ -454,6 +454,11 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.total_price){
|
||||
|
||||
+6
-6
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","3月总量","4月总量","5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","4月总量","5月总量","6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-03-01"
|
||||
def date = "2022-04-01"
|
||||
def endDate = "2022-05-01"
|
||||
def newDate = "2022-06-01"
|
||||
def startDate = "2022-04-01"
|
||||
def date = "2022-05-01"
|
||||
def endDate = "2022-06-01"
|
||||
def newDate = "2022-07-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -131,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-05成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-06成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
+38
-1
@@ -877,6 +877,8 @@ class CommonDatautils {
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
keys.put("sid", 1)
|
||||
keys.put("status", 1)
|
||||
|
||||
try{
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
@@ -2278,6 +2280,24 @@ class CommonDatautils {
|
||||
return prices;
|
||||
}
|
||||
|
||||
public static def queryDistrictPrices(city, region, startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)));
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
def prices = [];
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
def price = dbo.get("avg_price");
|
||||
if(price){
|
||||
prices << price;
|
||||
}
|
||||
}
|
||||
|
||||
return prices;
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPricesByDate(city, region, name, startDate, endDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
@@ -3031,9 +3051,26 @@ class CommonDatautils {
|
||||
return price;
|
||||
}
|
||||
|
||||
static def queryCityRegionPrices(city, region, startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("block", null);
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)));
|
||||
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def prices = [];
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
prices << dbo.get("avg_price");
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static def queryCityRegionBlockPrice(city, region, block, startDate, endDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
|
||||
+2
-1
@@ -129,7 +129,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
if(curPrice){
|
||||
m << dbo.get("_id").toString()//均价ID
|
||||
}else{
|
||||
m << ""
|
||||
m << new ObjectId().toString()
|
||||
}
|
||||
m << rMap.get("_id").toString()//楼盘ID
|
||||
m << rMap.city
|
||||
@@ -210,6 +210,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
def o = new BasicDBObject()
|
||||
println r[0]
|
||||
o.put("_id", new ObjectId(r[0]))
|
||||
o.put("comm_id", r[1])
|
||||
o.put("city", r[2])
|
||||
|
||||
+26
-9
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -55,11 +56,17 @@ class FzDataResultStat {
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
|
||||
def distkeys = []
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
map1.put(key, row[5])
|
||||
if(!distkeys.contains(key)){
|
||||
distkeys << key
|
||||
}else{
|
||||
println "重复:"+key
|
||||
}
|
||||
map1.put(key, row)
|
||||
}
|
||||
|
||||
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
@@ -67,19 +74,29 @@ class FzDataResultStat {
|
||||
for(def s : list2) {
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
map2.put(key, row[5])
|
||||
map2.put(key, row)
|
||||
}
|
||||
def rows = []
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
|
||||
map1.each {k,v->
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||
map1.each {k,list->
|
||||
def row = []
|
||||
def keys = k.split("##")
|
||||
def lastPrice = map2.get(k)
|
||||
def markup = null
|
||||
if(v && lastPrice){
|
||||
markup = (v.toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
|
||||
|
||||
def lastObj = map2.get(k)
|
||||
def lastPrice = null
|
||||
if(lastObj){
|
||||
lastPrice = lastObj[5]
|
||||
}
|
||||
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
|
||||
def markup = null
|
||||
if(list && lastPrice){
|
||||
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
|
||||
}
|
||||
def dist = CommonDatautils.findDistrictById(list[1])
|
||||
def flag = "有效"
|
||||
if(dist.status!=0){
|
||||
flag = "失效"
|
||||
}
|
||||
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
|
||||
+6
-5
@@ -73,8 +73,8 @@ class HouseExportBatch extends BaseBatch{
|
||||
def fileName = "/fzbank/output/FJW_HOUSE_"+curDate+".csv"
|
||||
def dataCodeFileName = "/fzbank/output/FJW_HOUSE_"+curDate+".sha512"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "F:\\数据\\福州\\FJW_HOUSE_"+curDate+".csv"
|
||||
dataCodeFileName = "F:\\数据\\福州\\FJW_HOUSE_"+curDate+".sha512"
|
||||
fileName = "D:\\数据\\福州\\FJW_HOUSE_"+curDate+".csv"
|
||||
dataCodeFileName = "D:\\数据\\福州\\FJW_HOUSE_"+curDate+".sha512"
|
||||
}
|
||||
batchCreateFile(ret, fileName, headers, charset);
|
||||
println "数据文件输出完成......"
|
||||
@@ -325,8 +325,8 @@ class HouseExportBatch extends BaseBatch{
|
||||
// int endDate = 20191201
|
||||
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyyMMdd")
|
||||
def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
|
||||
|
||||
// def endDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyyMMdd");
|
||||
def endDate = "20220710"
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
// QueryBuilder room_range = QueryBuilders.rangeQuery("room").from(1).to(5).includeLower(true).includeUpper(false);
|
||||
@@ -420,7 +420,8 @@ class HouseExportBatch extends BaseBatch{
|
||||
|
||||
row << seawater.name
|
||||
row << seawater.url
|
||||
row << seawater.p_date
|
||||
// row << seawater.p_date
|
||||
row << DateUtils.getRandomDate("2022-06-01", "2022-06-30")
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202205.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202206.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
@@ -136,8 +136,8 @@ def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
|
||||
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
|
||||
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
|
||||
println s
|
||||
//def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
//DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
|
||||
|
||||
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
|
||||
|
||||
+3
-3
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-03-01"
|
||||
static def statDate = "2022-06-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年7月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+2
-2
@@ -44,7 +44,7 @@ class AnjukeAppCrawl {
|
||||
// String line2 = StringEscapeUtils.unescapeJava(s);
|
||||
// println line2
|
||||
def task = new AnjukeAppCrawl();
|
||||
// task.autoRun()
|
||||
task.autoRun()
|
||||
// task.run();
|
||||
// task.run2()
|
||||
|
||||
@@ -55,7 +55,7 @@ class AnjukeAppCrawl {
|
||||
// task.groupByCity()
|
||||
// task.selectNoSave()
|
||||
// task.statCount()
|
||||
task.exportFzDist()
|
||||
// task.exportFzDist()
|
||||
}
|
||||
def saveToEs1(){
|
||||
def lists = getAllAppDatas()
|
||||
|
||||
+2
-2
@@ -25,11 +25,11 @@ class AnjukeHouseCrawl {
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
|
||||
def href = "https://fz.anjuke.com/prop/view/S2512363126645770"
|
||||
def href = "https://zj.lianjia.com/ershoufang/103120144204.html"
|
||||
def doc = getDocument(href, driver)
|
||||
// def doc = ClaABuYun.getResponse(href)
|
||||
// def newUrl = href.substring(0, hrefs.indexOf("?"))
|
||||
sendMq(href, doc, "sell_anjuke")
|
||||
sendMq(href, doc, "sell_lianjia")
|
||||
|
||||
// for(def i=25;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
|
||||
|
||||
+7
-3
@@ -10,7 +10,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class AnjukeSendMq {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ajk_community_test", "192.168.0.235", 27018)
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ajk_community_2207", "192.168.0.235", 27018)
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeSendMq();
|
||||
@@ -19,8 +19,12 @@ class AnjukeSendMq {
|
||||
|
||||
def run(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = collection.find(q).limit(30)
|
||||
q.put("city","¸£ÖÝ")
|
||||
def dbcursor = collection.find(q).batchSize(1000)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
sendMq(d.url, d.content, "community_anjuke")
|
||||
}
|
||||
}
|
||||
@@ -35,6 +39,6 @@ class AnjukeSendMq {
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
}
|
||||
}
|
||||
|
||||
+21
-11
@@ -7,6 +7,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
class RegionPriceExport {
|
||||
|
||||
@@ -18,25 +19,25 @@ class RegionPriceExport {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def months = ["20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
def regions = CommonUtils.getAllRegions("佛山","region")
|
||||
def months = ["20220101","20220201","20220301","20220401","20220501","20220601"]
|
||||
def regions = CommonUtils.getAllRegions("重庆","region")
|
||||
def rets = []
|
||||
rets << ["城市","区域","20210516","20210523","20210530","20210613","20210620","20210627","20210704","20210711"]
|
||||
rets << ["城市","区域","202201","202202","202203","202204","202205","202206"]
|
||||
regions.each{region,values->
|
||||
def data = []
|
||||
data << "佛山"
|
||||
data << "重庆"
|
||||
data << region
|
||||
for(def month :months){
|
||||
def dbo = findWeekendPrice("佛山", region, month);
|
||||
if(dbo){
|
||||
data << dbo.avg_price
|
||||
def price = findWeekendPrice("重庆", region, month);
|
||||
if(price){
|
||||
data << price
|
||||
}else{
|
||||
data << ""
|
||||
}
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\佛山区域均价数据.xlsx",rets, "价格")
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆区域均价数据.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
def findPrice(city, region, month){
|
||||
@@ -53,8 +54,17 @@ class RegionPriceExport {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("cat", "区域")
|
||||
q.put("weekend", DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
|
||||
def dbo = REGION_STAT.findOne(q)
|
||||
return dbo
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(month, "yyyyMMdd"))
|
||||
q.put("weekend", new BasicDBObject('$gte', startDate).append('$lte', endDate))
|
||||
def dbcursor = REGION_STAT.find(q)
|
||||
def prices = []
|
||||
for(def d : dbcursor){
|
||||
prices << d.avg_price
|
||||
}
|
||||
if(prices){
|
||||
return (prices.sum()/prices.size()).toInteger()
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -67,12 +67,12 @@ class FangjiaClientUtils {
|
||||
|
||||
|
||||
|
||||
public static def addressStand(def address){
|
||||
public static def addressStand(def city, def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
|
||||
+1
-2
@@ -71,8 +71,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
def run(){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
|
||||
//def citys = CommonDatautils.findAllCity();
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
|
||||
def code = cityCodeDatas.get(city)
|
||||
|
||||
+17
-4
@@ -17,6 +17,7 @@ import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.chrome.ChromeOptions
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
@@ -44,6 +45,7 @@ public class BeiKeCrawlByChrome {
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def citys = getAllCityUrls(driver)
|
||||
citys = citys.reverse()
|
||||
@@ -145,7 +147,12 @@ public class BeiKeCrawlByChrome {
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "/chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
try{
|
||||
driver.get(url)
|
||||
}catch(Exception e){
|
||||
println url +" 出错"
|
||||
}
|
||||
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
@@ -153,7 +160,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-04-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -190,10 +197,16 @@ public class BeiKeCrawlByChrome {
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
roomStr = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
if(roomStr && roomStr.isNumber()){
|
||||
detail.room = roomStr.toInteger()
|
||||
}
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
|
||||
def hallStr = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
if(hallStr && hallStr.isNumber()){
|
||||
detail.hall = hallStr.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+58
-11
@@ -15,28 +15,29 @@ class ExportPaimai {
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportPaimai();
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.addressStandard()
|
||||
// task.addPrice()
|
||||
// task.stat()
|
||||
task.stat()
|
||||
// task.standard()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def file = "D:\\数据\\重庆农商行\\法拍\\宿迁法拍数据.xlsx"
|
||||
def file = "D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已成交).xlsx"
|
||||
def datas = ExcelUtils.read(file, 0, 1)
|
||||
def index1 = 0
|
||||
|
||||
def rates1 = []
|
||||
for(def d : datas){
|
||||
if(d['P'] && d['P']=='Òѳɽ»'){
|
||||
if(d['M'] && d['M']=="住宅")
|
||||
// if(d['K']>=150){
|
||||
// if(d['M'] && d['M']=="住宅")
|
||||
// if(d['K']>=240){
|
||||
// index1++
|
||||
// def sumTradePrice = d['S']
|
||||
// def marketPrice = d['Y']
|
||||
// rates1 << sumTradePrice/marketPrice
|
||||
// }
|
||||
if(d['S']>2000000){
|
||||
if(d['S']<800000){
|
||||
index1++
|
||||
def sumTradePrice = d['S']
|
||||
def marketPrice = d['Y']
|
||||
@@ -56,12 +57,55 @@ class ExportPaimai {
|
||||
})
|
||||
}
|
||||
def addressStandard(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\重庆法拍数据.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\需要重跑地址解析和匹配_重庆法拍数据.xlsx",{d->
|
||||
def formatAddress = AddressUtil.formatAddress(d['C'], d['A'])
|
||||
if(formatAddress){
|
||||
d['F'] = formatAddress
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def standard(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\需要重跑地址解析和匹配_重庆法拍数据.xlsx",{d->
|
||||
def obj = com.fangjia.script.gj.district.FangjiaClientUtils.addressStand(d['C'], d['B'])
|
||||
if(obj){
|
||||
def arrs = obj.entities
|
||||
def address = ""
|
||||
def houseNum = ""
|
||||
def roomNum = ""
|
||||
def unitNum = ""
|
||||
def poi = ""
|
||||
for(def m : arrs){
|
||||
if(m.type == "road"){
|
||||
address = m.word
|
||||
}
|
||||
if(m.type == "roadno"){
|
||||
address = address + m.word
|
||||
}
|
||||
if(m.type == "houseno"){
|
||||
houseNum = m.word
|
||||
}
|
||||
if(m.type == "roomno"){
|
||||
roomNum = m.word
|
||||
}
|
||||
if(m.type == "cellno"){
|
||||
unitNum = m.word
|
||||
}
|
||||
|
||||
if(m.type == "poi"){
|
||||
poi = m.word
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
d['F'] = address
|
||||
d['G'] = houseNum
|
||||
d['H'] = unitNum
|
||||
d['I'] = roomNum
|
||||
d['J'] = poi
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -123,17 +167,20 @@ class ExportPaimai {
|
||||
row << d.url
|
||||
|
||||
rows << row
|
||||
}else{
|
||||
println d.title
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\重庆法拍原数据.xlsx",rows,"数据")
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已流拍).xlsx",rows,"数据")
|
||||
}
|
||||
|
||||
def findPaimaiData(){
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("city","重庆")
|
||||
pmquery.put("city","苏州")
|
||||
pmquery.put("region","姑苏")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-06-01")))
|
||||
pmquery.put("status","已成交")
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
|
||||
pmquery.put("status","已流拍")
|
||||
def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1))
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
|
||||
+6
-6
@@ -129,7 +129,7 @@ public class LianjiaCrawlByChrome {
|
||||
def standardRegion(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-03-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
@@ -153,7 +153,7 @@ public class LianjiaCrawlByChrome {
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-05-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
@@ -171,7 +171,7 @@ public class LianjiaCrawlByChrome {
|
||||
query.put("region", null)
|
||||
query.put("district_name", m.name)
|
||||
// query.put("source", "链家在线")
|
||||
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
|
||||
// query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
@@ -238,7 +238,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-04-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -442,11 +442,11 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["¸£ÖÝ"]
|
||||
def citys= ["°²Çì"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+21
-20
@@ -23,28 +23,29 @@ DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "p
|
||||
//String str = format.format(date);
|
||||
//System.out.println(str);
|
||||
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("city","ËÞǨ")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
//pmquery.put("status","Òѳɽ»")
|
||||
//pmquery.put("estateType", null)
|
||||
pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm")
|
||||
|
||||
def cur = paimai43_COLL.find(pmquery)
|
||||
def urls = []
|
||||
for(def o : cur){
|
||||
urls << o.url
|
||||
}
|
||||
println "´ý½âÎö£º"+urls
|
||||
//def pmquery = new BasicDBObject()
|
||||
//pmquery.put("city","ËÞǨ")
|
||||
//pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
////pmquery.put("status","Òѳɽ»")
|
||||
////pmquery.put("estateType", null)
|
||||
//pmquery.put("url","https://sf-item.taobao.com/sf_item/665574733237.htm")
|
||||
//
|
||||
//def cur = paimai43_COLL.find(pmquery)
|
||||
//def urls = []
|
||||
//for(def o : cur){
|
||||
// urls << o.url
|
||||
//}
|
||||
//println "´ý½âÎö£º"+urls
|
||||
def index = 0
|
||||
for(def url : urls){
|
||||
//for(def url : urls){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", url)
|
||||
// q.put("url", url)
|
||||
q.put("update_time", new BasicDBObject('$gte',DateUtils.parseFromTightByPattern("2022-07-12", "yyyy-MM-dd")))
|
||||
def dbcursor = collection.find(q)
|
||||
index++
|
||||
println index
|
||||
for(def d : dbcursor){
|
||||
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao")
|
||||
json.put("url", d.url)
|
||||
@@ -64,8 +65,8 @@ for(def url : urls){
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
}
|
||||
}
|
||||
//}
|
||||
|
||||
|
||||
|
||||
+33
-1
@@ -34,7 +34,39 @@ class PaimaiTest {
|
||||
// task.updatePaimai()
|
||||
// task.updateFieldPaimai()
|
||||
// task.statCity()
|
||||
task.jsStat()
|
||||
// task.jsStat()
|
||||
task.selectData()
|
||||
}
|
||||
|
||||
def selectData(){
|
||||
def urls = ["https://sf-item.taobao.com/sf_item/674045056937.htm",
|
||||
"https://sf-item.taobao.com/sf_item/669281206202.htm",
|
||||
"https://sf-item.taobao.com/sf_item/646818853936.htm",
|
||||
"https://sf-item.taobao.com/sf_item/636815760791.htm",
|
||||
"https://sf-item.taobao.com/sf_item/636055429237.htm",
|
||||
"https://sf-item.taobao.com/sf_item/635094754807.htm",
|
||||
"https://sf-item.taobao.com/sf_item/621286459023.htm",
|
||||
"https://sf-item.taobao.com/sf_item/613944729986.htm",
|
||||
"https://sf-item.taobao.com/sf_item/613944489517.htm",
|
||||
"https://sf-item.taobao.com/sf_item/604967216580.htm",
|
||||
"https://sf-item.taobao.com/sf_item/596302571187.htm",
|
||||
"https://sf-item.taobao.com/sf_item/594722220602.htm"
|
||||
]
|
||||
for(def url : urls){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", url)
|
||||
def obj = paimai43_COLL.findOne(q)
|
||||
if(obj){
|
||||
if(obj.matchName=="中渝国宾城"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("matchName", "中渝山顶道国宾城")
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
println url +" "+obj.matchName
|
||||
}else{
|
||||
println url +" 不存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def jsStat(){
|
||||
|
||||
@@ -1,17 +1,25 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import ch.qos.logback.core.db.dialect.DBUtil
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
|
||||
def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
|
||||
DBUtils.seaweed.save(dist)
|
||||
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
|
||||
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
//Date date = new Date(1617211269868);
|
||||
//print dateFormat.format(date);
|
||||
@@ -50,13 +58,13 @@ import java.text.SimpleDateFormat
|
||||
//}
|
||||
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"])
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
def year = CommonUtils.getDistrictAge(d)
|
||||
rets << [d.city,d.region, d.name, year]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据")
|
||||
//def dists = CommonDatautils.findAllDistrictByCity("ÉϺ£", "district", ["city","region","name","finish_building_date"])
|
||||
//def rets = []
|
||||
//for(def d : dists){
|
||||
// def year = CommonUtils.getDistrictAge(d)
|
||||
// rets << [d.city,d.region, d.name, year]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\Êý¾Ý\\ͳ¼Æ·¿Áä.xlsx", rets, "Êý¾Ý")
|
||||
|
||||
|
||||
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "Q·¿"//
|
||||
def citys = findCrawlCity(source, 202204);
|
||||
def source = "城市房产"//
|
||||
def citys = findCrawlCity(source, 202205);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202204);
|
||||
def count2 = findCount(source, city, 202203);
|
||||
def count1 = findCount(source, city, 202205);
|
||||
def count2 = findCount(source, city, 202206);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+1
-1
@@ -193,7 +193,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
// "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州",
|
||||
def citys = ["宁波","大连","佛山","乌鲁木齐","济南","长春","兰州","石家庄","温州","烟台","太原","泉州","南通","呼和浩特","徐州","惠州","厦门","南昌","临沂","贵阳","嘉兴","常州","盐城","威海","淄博","深圳","中山","重庆","苏州","西宁","绍兴","洛阳","廊坊","昆明","金华","珠海","宜昌","南充","信阳","杭州","沧州","宿迁","南阳","德州","桂林","新乡","包头","遵义","乐山","株洲","德阳","眉山","驻马店","许昌","宜宾","衡水","泸州","营口","清远","达州","宝鸡","长沙","广安"]
|
||||
def citys = ["¸£ÖÝ"]
|
||||
for(def city : citys){
|
||||
def str = cityList.get(city)
|
||||
for(def i=0;i<1;i++){
|
||||
|
||||
+13
-8
@@ -21,7 +21,9 @@ class SynchDistrictPrice {
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
def esSourceMap = ["ke.com":"±´¿Ç"]
|
||||
|
||||
//,"城市房产":"城市房产"
|
||||
def esSources = ["365淘房":"365淘房"]//"贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客PC端":"安居客","搜房":"搜房","吉屋":"吉屋"
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
// task.run()
|
||||
@@ -31,20 +33,20 @@ class SynchDistrictPrice {
|
||||
def run1(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
|
||||
esSourceMap.each { k, v ->
|
||||
esSources.each{esSource, mgSource->
|
||||
for(def city : citys){
|
||||
def rets = findAllEsData(city, v)
|
||||
def rets = findAllEsData(city, esSource, mgSource)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, "±´¿ÇÕÒ·¿")
|
||||
synchData(city, rets, mgSource)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findAllEsData(city,source){
|
||||
def findAllEsData(city,esSource, mgSource){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='±´¿Ç' and city='${city}'", {d->
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-06-15'", {d->
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
@@ -54,8 +56,11 @@ class SynchDistrictPrice {
|
||||
data.put("name", d.name)
|
||||
data.put("address", d.address)
|
||||
data.put("avg_price", d.avg_price)
|
||||
data.put("source", source)
|
||||
data.put("source", mgSource)
|
||||
data.put("s_date", SHORT_DATE)
|
||||
data.put("url", d.url)
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
rets << data
|
||||
}
|
||||
})
|
||||
@@ -138,7 +143,7 @@ class SynchDistrictPrice {
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("avg_price", r.avg_price)
|
||||
udata.put("m_date",new Date())
|
||||
thirdPartyColl.update(new BasicDBObject('_id',obj._id), udata)
|
||||
thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata))
|
||||
|
||||
}else{
|
||||
try{
|
||||
|
||||
Reference in New Issue
Block a user