From 9ed8f6dc21fc8bac36e4307352b024760526f2ae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 21 Jun 2023 15:10:18 +0800 Subject: [PATCH] update --- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 11 ++-- .../script/chongqing/HailanTest.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 27 +++++++++ .../script/district/MasterNameInAlias.groovy | 59 +++++++++++++++++++ .../fuzhou/DistrictInfoExportBatch.groovy | 2 + .../script/fuzhou/HouseExportBatch.groovy | 5 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 24 +++++--- 8 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index da8662b0e..492fb152d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); saveToPgSql(rows) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index e8bd49824..9156b9d6d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -45,6 +45,7 @@ class TradeStatLog { // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() + // task.deleteDistrict() // task.deleteDealData() // task.deleteDealData2() @@ -92,13 +93,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-02-01" - def date = "2023-03-01" - def endDate = "2023-04-01" - def newDate = "2023-05-01" + def startDate = "2023-03-01" + def date = "2023-04-01" + def endDate = "2023-05-01" + def newDate = "2023-06-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy index ff1b235f3..cdad0ddb6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -48,7 +48,7 @@ class HailanTest { } def run(){ - def inputfile = "D:\\数据\\微元素\\test.xlsx" + def inputfile = "D:\\数据\\微元素\\排查表.xlsx" // addThirdPrice(inputfile) // match(inputfile) // tradeAvgPrice(inputfile) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index bc5abebe7..e8cc8e9f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.commData +import com.fangjia.core.RegionData import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import groovy.json.JsonSlurper import net.sf.json.JSONArray @@ -3871,4 +3873,29 @@ class CommonDatautils { } + /** + * 根据经纬度获取区域 + * @param city + * @param location + * @return + */ + static def getRegionByLocation(city, def location){ + def json = GaoDeApiUtils.geocode(location); + def region = null; + if (json && json.info=="OK") { + def regeocode = json.regeocode + if(regeocode && regeocode.addressComponent){ + region = regeocode.addressComponent.district + if(!region){ + region = regeocode.addressComponent.township + } + } + } + if(region){ + region = RegionData.formatRegion(city, region) + } + return region; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy new file mode 100644 index 000000000..dd63a1c00 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.mongodb.BasicDBObject + +class MasterNameInAlias { + + static main(args) { + def task = new MasterNameInAlias() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def allList = [] + for(def city : citys){ + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"]) + + def cityList = [] + cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"] + for(def dist1 : disticts){ + def distList = [] + for(def dist2 : disticts){ + if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){ + if(dist2.alias && dist2.alias.contains(dist1.name)){ + def distance = '' + if(dist1.lng && dist2.lng){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d); + } + distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance] +// def newAlias = dist2.alias.remove(dist1.name) +// def d = new BasicDBObject() +// d.put("_id", dist2._id) +// +// def udata = new BasicDBObject() +// udata.put("alias", newAlias) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(d, new BasicDBObject('$set', udata)) + } + } + } + if(distList){ + cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', ''] + cityList.addAll(distList) +// cityList << [] + } + } + + if(cityList){ + allList.addAll(cityList) + println city +" "+cityList.size() + } + } + ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 464ac2301..eeabbfcae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.lang.StringUtils import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -321,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def s = str.toString(); index++; if(s && s!="null"){ + s = s.replace('"','\"') // if(s.toString().contains(",")){ // s = csvHandlerStr(s.toString()) // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 1dd114d8a..e4562341d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{ // sb.append(s+ "@|@") // } // } - sb.append(StringUtils.join(row.toArray(), "@|@")) + def str = StringUtils.join(row.toArray(), "@|@") + str = str.replace('"','\"') + sb.append(str) + sb.append(System.getProperty("line.separator")); println index diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 9ab7eacb3..07241353c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.paimai import com.fangjia.core.RegionData +import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.gj.ccb.FileUtils @@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -300,9 +301,7 @@ public class LianjiaCrawlByChrome { if(dealDate){ if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - println "1..."+new Date() def dbo = findTrade(durl , "链家在线") - println "2..."+new Date() if (!dbo) { parse(city, e, regionText, driver) } @@ -333,9 +332,7 @@ public class LianjiaCrawlByChrome { if(price){ def detailUrl = e.select(".title a").attr("href") driver.get(detailUrl) - println "3..."+new Date() def detailDoc = checkDocument(driver, detailUrl); - println "4..."+new Date() def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -353,6 +350,15 @@ public class LianjiaCrawlByChrome { if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } + if(!detail.region){ + def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1) + str = str.substring(0, str.indexOf("',")) + def locArrs = str.split(",") + def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble()) + if(gmap){ + detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat) + } + } def roomStr = arrs[1] if (roomStr) { @@ -427,9 +433,7 @@ public class LianjiaCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail - println "5..."+new Date() DEAL_PRICE_COLL.save(detail) - println "6..."+new Date() } } } @@ -581,9 +585,11 @@ public class LianjiaCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - elemUsername.sendKeys("15021384995"); +// elemUsername.sendKeys("15021384995"); +// elemPassword.sendKeys("12qwaszx"); - elemPassword.sendKeys("12qwaszx"); + elemUsername.sendKeys("16283166371"); + elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; Thread.sleep(15000)