From 8e027cceac1386f968d9ecb85c7afd62b451ed84 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jul 2023 14:06:35 +0800 Subject: [PATCH] update --- .../BigdataCommunityBatch.groovy | 21 ++++++--- .../ccbevaluation/BigdataForsaleBatch.groovy | 9 +++- .../fangjia/script/chongqing/PocTest.groovy | 31 ++++++++++--- .../script/commData/CommonDatautils.groovy | 44 +++++++++---------- .../fuzhou/DistrictInfoExportBatch.groovy | 18 ++------ .../fuzhou/DistrictPriceExportBatch.groovy | 2 +- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../script/gj/crawl/OfficeProcess.groovy | 8 ++-- .../gj/paimai/BeiKeCrawlByChrome.groovy | 6 ++- 9 files changed, 86 insertions(+), 57 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 61b833dab..49d4de707 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -49,15 +49,25 @@ class BigdataCommunityBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def index = 0 + def today = new Date().getDate() + println "starting 。。。。。。。。。" for(def city : citys){ index ++ + + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 -// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + + def ccbMaps = null + if(today<=10){ + ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 + }else{ + ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + } + def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -77,9 +87,10 @@ class BigdataCommunityBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成->导Oracle" - delPgData() - saveToPgSql(rows) - + if(today > 10){ + delPgData() + saveToPgSql(rows) + } // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") // delPgData() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 3479e90ea..10822463f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -69,6 +69,10 @@ class BigdataForsaleBatch extends BaseBatch{ def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") + + def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); + def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"); + static main(args) { runBatch(new BigdataForsaleBatch(), args); } @@ -81,6 +85,9 @@ class BigdataForsaleBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def startDate = lastMonth+"18" + def endDate = curMonth+"19" + println "房源时间:"+startDate+" "+endDate def houseIdsfilter = findAllHouseIdsByCity() @@ -91,7 +98,7 @@ class BigdataForsaleBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy index 29a2403b5..a596bbb54 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy @@ -20,6 +20,25 @@ class PocTest { static main(args) { def task = new PocTest(); task.run(); +// task.export() + } + + def export(){ + ExcelUtils.modify("D:\\数据\\其它\\广州.xlsx", { row -> + def dist = CommonDatautils.findDistrictById(row["B"]) + row['F'] = dist.address + row['G'] = dist.lng/10000000000.0 + row['H'] = dist.lat/10000000000.0 + row['I'] = dist.estate_type2 + row['J'] = dist.build_type + row['K'] = dist.schools + row['L'] = dist.kindergartens + row['M'] = dist.loudong_count + row['N'] = dist.household_count + row['O'] = dist.estate_charge + row['P'] = dist.finish_building_date + row['Q'] = dist.estate_company + }) } @@ -32,9 +51,10 @@ class PocTest { * @return */ def run(){ - def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls" +// def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls" + def inputfile = "D:\\数据\\其它\\广州.xlsx" // match(inputfile) -// addPrice(inputfile) + addPrice(inputfile) // addArroundPrice(inputfile) // standard(inputfile) // addLoudongNum(inputfile) @@ -132,15 +152,16 @@ class PocTest { def addPrice(path){ ExcelUtils.multiThreadingModify(path, { row -> - def commid = row['R'] + def commid = row['B'] if(commid){ def dist = CommonDatautils.findDistrictById(commid.trim()) if(dist){ - row['Q'] = dist.name +// row['Q'] = dist.name def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) if(priceObj){ - row['F'] = priceObj.avg_price + row['R'] = priceObj.avg_price } + row['S'] = CommonDatautils.findNewSqmetreRent(dist.city, dist.region, dist.name) } } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 4540e31c5..454b1a9e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -252,7 +252,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "鲁能大厦") +// q.put("name", "交发风景") DBObject keys = new BasicDBObject(); if(fields){ @@ -899,28 +899,28 @@ class CommonDatautils { DBObject q = new BasicDBObject(); q.put("_id", new ObjectId(id.toString().replace("[","").replace("]",""))); - DBObject keys = new BasicDBObject(); - keys.put("alias", 1); - keys.put("addr_alias", 1); - keys.put("city", 1); - keys.put("region", 1); - keys.put("name", 1); - keys.put("estate_type2", 1); - keys.put("address", 1); - keys.put("build_structure", 1); - keys.put("out_pics", 1); - keys.put("lat", 1); - keys.put("lng", 1); - keys.put("lat2", 1); - keys.put("lng2", 1); - keys.put("name_cat", 1); - keys.put("finish_building_date", 1) - keys.put("mapping_alias", 1) - keys.put("sid", 1) - keys.put("status", 1) - keys.put("name_cat", 1) +// DBObject keys = new BasicDBObject(); +// keys.put("alias", 1); +// keys.put("addr_alias", 1); +// keys.put("city", 1); +// keys.put("region", 1); +// keys.put("name", 1); +// keys.put("estate_type2", 1); +// keys.put("address", 1); +// keys.put("build_structure", 1); +// keys.put("out_pics", 1); +// keys.put("lat", 1); +// keys.put("lng", 1); +// keys.put("lat2", 1); +// keys.put("lng2", 1); +// keys.put("name_cat", 1); +// keys.put("finish_building_date", 1) +// keys.put("mapping_alias", 1) +// keys.put("sid", 1) +// keys.put("status", 1) +// keys.put("name_cat", 1) try{ - DSCursor cursor = getDataStoreConnection().find(q, keys); + DSCursor cursor = getDataStoreConnection().find(q); if(cursor){ for(def obj : cursor){ return obj; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 75ff94eee..414e61113 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -74,29 +74,17 @@ class DistrictInfoExportBatch extends BaseBatch{ def createDatas(def city){ def result = [] - def distKeys = [] def districts = CommonDatautils.findAllDistricts(city, "district", null); - for(def dist : districts){ - def key = dist.city +"##"+dist.region+"##"+dist.name - if(!distKeys.contains(key)){ - distKeys << key - } - } + def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); -// for(def dist : removeDists){ -// def key = dist.city +"##"+dist.region+"##"+dist.name -// if(!distKeys.contains(key)){ -// distKeys << key -// districts << dist -// } -// } + def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) def startDate = DateUtils.getFirstDayOfMonth(lastDate); def fzAddrMap = getFzAddress(); //因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据 -// saveOrUpdateFzDist(districts) + saveOrUpdateFzDist(districts) remarkRemoveDist(removeDists) def fzDistricts = getFzDistricts(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 2dbc3b908..2aa5282a0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{ static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") - def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] + def specialDistsPriceMap = ["津泰新村9-13号、15-18号":54000,"津泰新村14号":72000,"津泰新村1-6号、8号":35000,"津泰新村7号":75000,"津泰新村19号":25000] static main(args) { runBatch(new DistrictPriceExportBatch(), args) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 4de90b96b..ac0ef538c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-05-01" + static def statDate = "2023-06-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy index 55ce0ed8f..d7fd9e44a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy @@ -435,7 +435,7 @@ class OfficeProcess { pstmt.setString(5, d.source) println count+" "+d.city+" "+d.name pstmt.addBatch() - if(count % 1 ==0) { + if(count % 100 ==0) { try { pstmt.executeBatch() } catch (Exception e) { @@ -480,7 +480,7 @@ class OfficeProcess { } def checkRegion(){ - def citys = ["杭州","苏州"]//"上海","深圳","北京","广州","重庆","成都", + def citys = ["上海"]//"上海","深圳","北京","广州","重庆","成都", for(def city : citys){ def regionPointsMap = findRegionRange(city) def addDatas = findAddDatasByClickhouse(city) @@ -492,7 +492,7 @@ class OfficeProcess { if(data.lng){ def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region)) def mark = flag?'T':'F' -// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark] + rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark] updateOption(data.city, data.region, data.name, data.source, mark) println index+" "+data.region + " "+data.name+" "+ flag } @@ -502,7 +502,7 @@ class OfficeProcess { } def findAddDatasByClickhouse(city){ - def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%添加%'" + def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and region_fence is null" def datas = [] sqlInstance.eachRow(sql, {r-> datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 0d031c591..2691fe7de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -490,9 +490,11 @@ public class BeiKeCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - elemUsername.sendKeys("15021384995"); +// elemUsername.sendKeys("15021384995"); +// elemPassword.sendKeys("12qwaszx"); - elemPassword.sendKeys("12qwaszx"); + elemUsername.sendKeys("16283166371"); + elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; Thread.sleep(10000)