From 9100314a96e2591ce454eaf7070c8d3e9c3c2a48 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jan 2022 14:30:39 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=88=BF=E6=BA=90=E6=97=B6?= =?UTF-8?q?=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/ExportOracleData.groovy | 11 +- .../fangjia/script/ccbevaluation/Test.groovy | 88 +++- .../script/ccbevaluation/TradeStatLog.groovy | 16 +- .../ccbevaluation/data/CcbDistrictToDB.groovy | 89 +++- .../ccbevaluation/data/ExcelDataTest.groovy | 119 ++++- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../script/gj/level/DistrictIndexBatch.groovy | 9 +- .../script/gj/paimai/ChromeDriverPool.groovy | 61 +++ .../gj/paimai/LianjiaCrawlByChrome.groovy | 428 ++++++++++++++++++ .../fangjia/script/gj/paimai/Worker.groovy | 31 ++ .../script/thirdPriceCrawl/Esf58Crawl.groovy | 54 +-- .../trade/LianJiaTradeCrawl.groovy | 185 ++++++++ .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- 14 files changed, 1007 insertions(+), 94 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 41273e6c3..eb62806e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211118, 20211219); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 736221526..bc608f48f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -11,7 +11,7 @@ import java.sql.ResultSet class ExportOracleData { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -24,11 +24,11 @@ class ExportOracleData { } def run() { - def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // exportGuidedPrice() - exportCommunityGuidedPrice(lastMonth.toString()+"28") +// exportCommunityGuidedPrice(lastMonth.toString()+"28") // // //exportThirdPartyPrice()//废弃掉,老表 // exportThirdPartyPrice2(lastMonth) @@ -47,12 +47,13 @@ class ExportOracleData { // exportAvmEvaluationGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth) -// exportCommunityBasePrice(curMonth) + // // exportCityPrice(curMonth) // exportRegionPrice(curMonth) -// exportCommunityBasePriceGj(curMonth) +// exportCommunityBasePriceGj(curMonth)//已合下面的合并 + exportCommunityBasePrice(curMonth) // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0707c749c..15f1d38c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -27,8 +27,8 @@ import java.util.concurrent.atomic.AtomicLong import java.util.regex.*; -def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -271,7 +271,7 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //},0) //def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充数据删除3658_1228.xlsx",{d-> +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d-> // if(d['A']){ // def udata = new BasicDBObject() // udata.put("name_cat", 11) @@ -284,26 +284,26 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // println index // } //},0) -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> - if(d['A']){ - def region = d['B'] - def newRegion = RegionData.formatRegion(d['A'], region) - if(newRegion) { - def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } - } - } -},0) +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> +// if(d['A']){ +// def region = d['B'] +// def newRegion = RegionData.formatRegion(d['A'], region) +// if(newRegion) { +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +// if(dist){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dist._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d +// } +// } +// } +//},0) //def citys = CcbCommonUtils.citys //def rets = [] @@ -488,6 +488,48 @@ ExcelUtils.modify("D:\\ // } //},0) +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// if(d['K']){ +// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) +// def address = null +// if(formatAddress && formatAddress.endsWith("号")) { +// address = formatAddress +// } +// +// +// def data = new BasicDBObject(); +// data.put("m_date", new Date()) +// if(d['M']){ +// long glat = d['M'].toString().toDouble() * 10000000000L +// long glng = d['L'].toString().toDouble() * 10000000000L +// data.put("lat",glat) +// data.put("lng",glng) +// +// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) +// if(baidu){ +// long blng = baidu.x * 10000000000L +// long blat = baidu.y * 10000000000L +// data.put("lat2",blat) +// data.put("lng2",blng) +// } +// +// } +// +// if(address){ +// data.put("address", address) +// } +// index++ +// println index +" "+d['B'] + " "+data +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) +// } +// } +// } +//}) + def findDealCount(sqlInstance, def startDate, def endDate){ def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate + def city = d['A'] + def region = d['B'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['C'] + def alias = [] + if(d['F']){ + if(name != d['F']){ + alias << d['F'] + } + } + + def formatAddress = null + if(d['G']){ + formatAddress = AddressUtil.formatAddress(d['A'], d['G']) + } + def address = null + if(d['D']){ + address = d['D'] + } + if(d['E']){ + address = address + d['E'] + } + + if(!formatAddress && address){ + formatAddress = AddressUtil.formatAddress(d['A'], address) + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['H']){ + glat = d['I'].toString().toDouble() * 10000000000L + glng = d['H'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['H'].toString().toDouble(), d['I'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + } + + if(name){ + def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district") + if(!dist){ + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", d['A']) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + if(alias){ + data.put("alias",alias) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + } + } + } +// } + },0); } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy index 8ec8ba66a..e7411bf1b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -15,7 +15,7 @@ import com.mongodb.DBCollection DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d -> // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion && !d['K']) { // def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']); @@ -32,7 +32,7 @@ DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price" //}); -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d-> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d-> // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion && !d['K']){ // @@ -63,7 +63,7 @@ DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price" // //},0) -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d-> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d-> // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion && !d['K'] && d['G']){ // @@ -303,27 +303,100 @@ def city = " //}); -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> - if(d['K'] && d['K']=="别名(高德)" && d['G']){ +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +// if(d['K'] && d['K']=="别名(高德)" && d['G']){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district") +// def dist = null +// if(rets){ +// dist = rets[0] +// } +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['C'])){ +// alias << d['C'] +// } +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// } +// } +//}); + + + +//def rets = [] +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d -> +// if(d['A']){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['E'])){ +// alias << d['E'] +// } +// if(alias && dist.mapping_alias!=alias){ +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// rets << [dist._id.toString(), d['E']] +// } +// } +// } +//}) +//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据") + + +def rets = [] +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d -> +// if(d['A']){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// def dists = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district") +// def dist = null +// if(dists && dists.size()==1){ +// dist = dists[0] +// } +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['D'])){ +// alias << d['D'] +// } +// if(alias && dist.mapping_alias!=alias){ +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// rets << [dist._id.toString(), d['D']] +// } +// } +// } +//}) +//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据") + + +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d -> + if(d['A']) { def newRegion = RegionData.formatRegion(d['A'], d['B']) - if(newRegion){ - def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district") - def dist = null - if(rets){ - dist = rets[0] - } - if(dist){ - def alias = dist.mapping_alias - if(!alias){ - alias = [] - } - if(!alias.contains(d['C'])){ - alias << d['C'] - } - def udata = new BasicDBObject() - udata.put("mapping_alias",alias) - DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + if (newRegion) { + def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district") + if (dist) { + rets << [dist._id.toString(), d['C']] } } } -}); +}) +ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index a1350cc2f..c43936a8a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,7 +48,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-11-01" + static def statDate = "2021-12-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年12月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index e0460cf88..05ed58021 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -244,9 +244,12 @@ class DistrictIndexBatch extends BaseBatch{ } } } - def avg_charge = rets.sum()/rets.size(); - avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge) - return avg_charge; + if(rets){ + def avg_charge = rets.sum()/rets.size(); + avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge) + return avg_charge; + } + return null } def getPriceInRegionRank(d, nbLists, priceMap){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy new file mode 100644 index 000000000..53c9378bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.gj.paimai + +import org.openqa.selenium.chrome.ChromeDriver + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +public class ChromeDriverPool { + + private static ConcurrentHashMap pool ;//线程池 + private static int coreSize = 2;//核心线程池,一直驻留的资源 + private static int maxSize = 10;//TODO 设置最大数量资源,用于解决瞬时并发,瞬时并发过后要定时去识别空闲的释放掉,保留coreSize的资源 + private static AtomicInteger count = new AtomicInteger(0);//原子自增!! + + static { + //驱动初始化 + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + pool = new ConcurrentHashMap<>(); + } + + private ChromeDriverPool(){ + } + + public static void init(int coreSize){//线程池数量初始化 + ChromeDriverPool.coreSize = coreSize; + } + + public static Worker getPool(){ + if(count.getAndIncrement() < coreSize){//自增!!! + return push(new Worker()); + }else{ + return pop(); + } + } + + + public static Worker push(Worker worker){ + pool.put(count.get(),worker); + return worker; + } + + /** + * 循环获取未使用的资源 + * @return + */ + public static Worker pop(){ + Set> sets = pool.entrySet(); + Iterator> its = sets.iterator(); + while(its.hasNext()){ + Map.Entry workEntry = its.next(); + Worker work = (Worker)workEntry.getValue(); + if(work.status == 1){ + return work; + } + } + return null; + } + + +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy new file mode 100644 index 000000000..cba9e27b7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -0,0 +1,428 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.gj.paimai.ChromeDriverPool +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +public class LianjiaCrawlByChrome { + + def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new LianjiaCrawlByChrome(); + task.run() + // task.run1() + } + + def run1(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def cityUrls = getAllCityUrls(driver) + getLogin(driver) + + for(def cityUrl : cityUrls){ + def detailUrls = getPageUrls(driver, cityUrl) + println cityUrl +" 详情链接:"+detailUrls.size() + poolCrawl( detailUrls) + } + } + + def poolCrawl(detailUrls){ + //模拟并发 + int parallelSize = 10; + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService pool = Executors.newFixedThreadPool(parallelSize); + + for (int i = 0; i < detailUrls.size(); i++) { + final def detailUrl = detailUrls[i] + pool.execute(new Runnable() { + @Override + public void run() { + + Worker work = null; + try { + latch.await();//阻塞所有线程 + System.out.println(Thread.currentThread().getName()); + while (true) {//业务需求,一直等待直到获取到驱动 + work = ChromeDriverPool.getPool(); + if (work != null) { + break; + } + Thread.sleep(1000); + } + work.start(); + ChromeDriver driver = work.getDriver(); + def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl) + println detail.city + " " + detail.region + " " + detail.district_name + System.out.println(Thread.currentThread().getName()+":"+driver); + Thread.sleep(1000 * 10); + } catch (Exception e) { + System.out.println(e.getMessage()); + }finally { + work.shutdown(); + } + } + }); + } + latch.countDown(); + } + + def run(){ +// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); +// System.setProperty("http.nonProxyHosts", "localhost"); +// WebDriver driver = new ChromeDriver() +// def cityMap = getAllCityUrls(driver) +// getLogin(driver) +// crawlData(driver, cityMap) + + addDealRegion(); + } + + def addDealRegion(){ + def q = new BasicDBObject() + q.put("region", null) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = DEAL_PRICE_COLL.find(q) + def keys = [] + for(def d : dbcursor){ + keys << ["city":d.city, "name":d.district_name] + } + for(def m : keys){ + //def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district") + def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district") + if(list && list.size()==1){ + def dist = list[0] + + def query = new BasicDBObject() + query.put("city", m.city) + query.put("district_name", m.name) + query.put("source", "链家在线") + query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) + + def data = new BasicDBObject() + data.put("region", dist.region) + def count = DEAL_PRICE_COLL.count(query) + DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) + } + } + } + + def crawlData(driver, cityMap){ + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + cityMap.each{city, cityUrl-> + //def detailUrls = getPageUrls(driver, cityUrl) + def detailUrls = getPageDeail(driver, city, cityUrl) + println cityUrl +" 详情链接:"+detailUrls.size() + // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) + } + } + + def getPageUrls(driver, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + }else{ + stop = true; + break; + } + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + + def getPageDeail(driver, city, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + parse(city, e) + }else{ + stop = true; + break; + } + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + def parse(city,e){ + try { + def title = e.select(".title a").text() + if (title) { + def detail = new BasicDBObject() + def arrs = title.split(" ") + detail.city = city + detail.district_name = arrs[0] + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + + def roomStr = arrs[1] + if (roomStr) { + if (roomStr.indexOf("室") > 0) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + } + if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + } + } + + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + def price = e.select(".totalPrice .number").text() + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = e.select(".title a").attr("href") + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } + } + }catch(Exception ex){ + ex.printStackTrace() + println e.select(".title a").attr("href") + } + } + + + def findRegionByThirdPary(city, district_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("name", district_name) + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1) + def dbo = null; + for(def d : dbcursor){ + dbo = d + } + if(dbo){ + return dbo.region + } + return null; + } + def getContent(driver, detailUrl){ + driver.get(detailUrl) + def str = driver.getPageSource() + if(str && str.contains("登录")){ + getLogin(driver) + return getContent(driver, detailUrl) + } + return str; + } + def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){ + if(detailUrls){ + for(def i=0;i=4){ + detail.city = eles[0].text().replace("房产网","") + detail.region = eles[2].text().replace("二手房成交","") + detail.district_name = eles[eles.size()-1].text().replace("二手房成交","") + } + if(detail.district_name) { + def price = JQueryUtils.selectText(".dealTotalPrice i", doc) + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + def dbo = findTrade(DEAL_PRICE_COLL, detail) + if (!dbo) { + def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") + if (roomStr) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")) + } + detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") + detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") + def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "") + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = detailUrl + detail.c_date = new Date() + detail.m_date = new Date() + + DEAL_PRICE_COLL.save(detail) + } + } + return detail; + } + + + def findTrade(def detail){ + def q = new BasicDBObject() + q.put("city", detail.city) + q.put("region", detail.region) + q.put("district_name", detail.district_name) + q.put("area", detail.area) + q.put("total_price", detail.total_price) + q.put("trade_date", detail.trade_date) + q.put("source", detail.source) + def dbo = DEAL_PRICE_COLL.findOne(q) + return dbo + } + + def getAllCityUrls(driver){ + driver.get("https://www.lianjia.com/city/") + Thread.sleep(8000) + def cityDoc = Jsoup.parse(driver.getPageSource()); + def cityEles = JQueryUtils.select(".city_province a", cityDoc) + def cityMap = [:] + if(cityEles){ + for(def e : cityEles){ + def href = e.attr("href") + if(!href.contains("fang.lianjia")){ + cityMap.put(e.text(), href) + } + } + } + return cityMap + } + + def getLogin(driver){ + driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); + WebDriverWait wait = new WebDriverWait(driver,20) + Thread.sleep(10000) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + + + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + + + elemUsername.sendKeys("15021384995"); + + elemPassword.sendKeys("12qwaszx"); + + btn.submit(); // 点击行为; + } +} + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy new file mode 100644 index 000000000..72d9c4559 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy @@ -0,0 +1,31 @@ +package com.fangjia.script.gj.paimai + +import org.openqa.selenium.chrome.ChromeDriver + +class Worker { + private ChromeDriver driver; + /** + * 状态修改要让其他线程可见 + * 1就绪 2运行 + */ + private volatile int status; + + public Worker(){ + driver = new ChromeDriver(); + status = 1; + } + + + public void shutdown(){ + this.status = 1; + } + + + public void start(){ + this.status = 2; + } + + public ChromeDriver getDriver() { + return driver; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 10c39958e..46b9b6d4c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -42,35 +42,38 @@ class Esf58Crawl extends BaseBatch{ //"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", // "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", // "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", -// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","三亚":"sanya|2422","阜阳":"fy|2325", +// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", // "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", -// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385", - def cityList = ["北京":"bj|1","上海":"sh|2","深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", - "宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", - "淮北":"huaibei|9357","滁州":"chuzhou|10266", - "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", - "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", - "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", - "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", +// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", +// "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", +// "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", +// "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", +// "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", +// "淮北":"huaibei|9357","滁州":"chuzhou|10266", +// "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", +// "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", +// "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", +// "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752", + def cityList = ["武夷山":"wuyishan|10761", "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", - "漳浦":"zhangpu|717", "珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791", - "肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303", + "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", + "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", - "南宁":"nn|845","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", + "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", - "遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", + "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", - "海口":"haikou|2053","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", + "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", - "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380", + "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", @@ -82,7 +85,7 @@ class Esf58Crawl extends BaseBatch{ "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", - "哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", + "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", @@ -93,21 +96,21 @@ class Esf58Crawl extends BaseBatch{ "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", - "长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914", - "湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", + "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", + "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", - "保定":"bd|424","廊坊":"lf|772","秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993", + "邢台":"xt|751","衡水":"hs|993", "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", - "沙河":"shaheshi|755", "徐州":"xz|471", "扬州":"yz|637","淮安":"ha|968", - "泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", + "沙河":"shaheshi|755", + "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", @@ -123,7 +126,7 @@ class Esf58Crawl extends BaseBatch{ "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", "梨树县":"lishu|10176", - "鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722", + "鞍山":"as|523","抚顺":"fushun|5722", "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", @@ -131,10 +134,10 @@ class Esf58Crawl extends BaseBatch{ "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", - "西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", + "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", "泰安":"ta|686","聊城":"lc|882", - "枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","菏泽":"heze|5632", + "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", @@ -170,8 +173,7 @@ class Esf58Crawl extends BaseBatch{ "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", - "宁波":"nb|135","金华":"jh|531", - "绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793", + "丽水":"lishui|7921","衢州":"quzhou|6793", "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy new file mode 100644 index 000000000..6540e9ca0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.thirdPriceCrawl.trade + +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.HttpClient +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.apache.http.Header +import org.apache.http.HttpResponse +import org.apache.http.client.methods.HttpGet +import org.apache.http.impl.client.HttpClients +import org.apache.http.client.HttpClient + +class LianJiaTradeCrawl { + + def static final LOGIN_URL = "https://clogin.lianjia.com/authentication/authenticate" + + public def CHARSET = "utf-8"; + public def ERROR_MSG = "访问过于频繁"; + + static main(args) { + def task = new LianJiaTradeCrawl() + task.run() + } + + def run(){ + def valueMap = getUidAndSid(); + def xqMap = goToXiaoqu(valueMap); + // goLogin() + } + + def getUidAndSid(){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("https://sh.lianjia.com/"); + get.setHeader("Accept-Encoding", "gzip, deflate, br") + get.setHeader("Accept-Language", "zh-CN,zh;q=0.9") + get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + HttpResponse response = client.execute(get); + def valueMap = getUidAndSid(response) + return valueMap; + + } + + def goToXiaoqu(valueMap){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("https://sh.lianjia.com/xiaoqu/"); + get.setHeader("Host","sh.lianjia.com") + get.setHeader("Connection","keep-alive") + get.setHeader("sec-ch-ua-mobile","?0") + get.setHeader("sec-ch-ua-platform","Windows") + get.setHeader("Upgrade-Insecure-Requests","1") + get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + get.setHeader("Sec-Fetch-Site","same-origin") + get.setHeader("Sec-Fetch-Mode","navigate") + get.setHeader("Sec-Fetch-User","?1") + get.setHeader("Sec-Fetch-Dest","document") + get.setHeader("Referer","https://sh.lianjia.com/") + get.setHeader("Accept-Encoding","gzip, deflate, br") + get.setHeader("Accept-Language","zh-CN,zh;q=0.9") + + def cookieStr = "" + valueMap.each {k,v-> + cookieStr = cookieStr +k +"="+v+";" + } + get.setHeader("Cookie", cookieStr) + + HttpResponse response = client.execute(get); + def newVvalueMap = getUidAndSid(response) + return valueMap; + } + + + def getUidAndSid(HttpResponse response) { + Header[] headers = response.getHeaders("Set-Cookie"); + def valueMap = [:] + for(def header : headers){ + String[] split = header.getValue().split(";"); + for (String s : split) { + def arrs = s.split("=") + valueMap.put(arrs[0], arrs[1]) + } + } + return valueMap; + } + + + + def goLogin() { + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","clogin.lianjia.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","1415") + headers.put("sec-ch-ua",'Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"') + headers.put("sec-ch-ua-mobile","?0") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + headers.put("sec-ch-ua-platform","Windows") + headers.put("Content-Type","application/json") + headers.put("Accept","*/*") + headers.put("Sec-Fetch-Site","same-origin") + headers.put("Sec-Fetch-Mode","cors") + headers.put("Sec-Fetch-Dest","empty") + headers.put("Cookie","Cookie: lianjia_uuid=45384ffe-4245-46fb-aa3f-aa6931a0a392; lianjia_ssid=75aa3a46-9d5f-42a9-b39b-4fc5e181e37c; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22%24device_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; select_city=350100; crosSdkDT2019DeviceId=-y4aedm--2s8wtc-85pqx15zlsvkewy-gkpcb9u2t") + + headers.put("Accept-Encoding", "gzip, deflate, br") + headers.put("Accept-Language", "zh-CN,zh;q=0.9") +// headers.put("User-Agent", UserAgents.get()) + headers.put("Referer", "https://clogin.lianjia.com/authentication/sdk/init?version=2.0") + headers.put("Origin", "https://clogin.lianjia.com") + //params信息 + Map params = new LinkedHashMap() + params.put("mainAuthMethodName", "username-password") + params.put("srcId", "eyJ0Ijoie1wiZGF0YVwiOlwiMWM1ZDg5ZTFjOTc0ZWViODliOTJkYTE1ZWEzOWM5ZDY1MWNlODJkMjkzYzQzYmY2NjRiMWEyYWI2NDRkYmViY2RiMDVkNmY3YTc3MmVkODBlZjNhMjBjYWM5ODVlNjY0ODc3YmVjMTBjYTRkM2RkNWFhOTY0Mjc0OTliNzVkZWU4ZDU1YzVkNzA1ZTRhMzcyY2U1MGZkMGVmZDE0OWY0NzYxMzk4MjRlMjU0Yzk0MzY0MDJjMjdkYjkxODdjNDliOWU4MWJlNWJjNGY2ZmEyNmY5Yjk2NzAxOTFkZDQwOWMxMzhjNTEwOTUwMmQ1OGE4NTE0Y2IzMzgxNzQ0ZTgwMzg2NmVjOGZhOWIyNjc1MzEyMTQ5YjE2NjU3ZjEyNDYyY2RiOTc4ZjM2ODVlNWVmYWViMGU1YjEwZmFiMWUzNDFhMzg4NmYzZmMwM2FhZWNjZDg1ZmIwNGYzZDQ2NmRjNlwiLFwia2V5X2lkXCI6XCIxXCIsXCJzaWduXCI6XCIzYjYwYTYwOFwifSIsInIiOiJodHRwczovL2Nsb2dpbi5rZS5jb20vbG9naW4/c2VydmljZT1odHRwcyUzQSUyRiUyRnd3dy5rZS5jb20lMkZ1c2VyJTJGY2hlY2tsb2dpbiUzRnJlZGlyZWN0JTNEaHR0cHMlMjUzQSUyNTJGJTI1MkZmei5rZS5jb20lMjUyRmNoZW5namlhbyUyNTJGYzM0MjAwMzI1MDU1MzAxMDElMjUyRiIsIm9zIjoid2ViIiwidiI6IjAuMSJ9") + params.put("version", "2.0") + params.put("accountSystem", "customer") + params.put("service", "https://www.lianjia.com/user/checklogin?redirect=https%3A%2F%2Ffz.lianjia.com%2Fchengjiao%2F") + params.put("loginTicketId", "nM7eXE8ZWBP67aiapHqZRpn0QHTfCX4q") + + + Map credentialparams = new LinkedHashMap() + credentialparams.put("username", "15021384995") + credentialparams.put("password", "dx/vjEHyEc8ETOqKDn5ZhNQZuO5JjoCu1D0UXVTQ2UrOE/liQYeNcj9H9bPB5c43UDRyKZSvA9UzBc4ycTOkDkq7pgITkJrzP0Iagkj19R0TT/yX/9NHEwm/GaqBfVvfJXfmw6ZX9vCYLaOaiOJ31PTpI5hDUS/PxxHBkmqC7Bg=") + credentialparams.put("ioaRunState", false) + credentialparams.put("ioaMid", "{}") + credentialparams.put("alertSt", "") + credentialparams.put("ioaStateVersion", 2) + credentialparams.put("encodeVersion", "1") + params.put("credential", credentialparams) + + Map contextparams = new LinkedHashMap() + contextparams.put("deviceId", "default") + contextparams.put("sign", "default") + params.put("context", contextparams) + + //post请求 + responseMap = isProxyAndTryTimesToRequest(LOGIN_URL, HttpClient.CHARSET, 100, "POST", params, headers, true) + } catch (Exception e) { +// println "goLogin(${username}, ${password}, ${rememberme})发生异常, 异常信息为:" + e.getMessage() + } + return responseMap + } + + + def static isProxyAndTryTimesToRequest(url, charset, tryTimes, requestType, params, headers, isJson) { + def responseMap = [:] + def _tryTimes_ = 0 + //请求响应为空(切换代理重试) + while (_tryTimes_ < tryTimes) { + try { + //第一次请求不提示重试 +// println("请求地址URL为:" + url + "代理尝试次数tryTimes为" + (_tryTimes_ + 1)) + if (StringUtils.isNotEmpty(requestType) && requestType.equals("POST")) { + responseMap = ABuYunUtil.doPostResponse(url, params, headers, charset, isJson) + } else { + responseMap = ABuYunUtil.doGetResponse(url, params, headers, charset) + } + def response = responseMap.get("response") + if (StringUtils.isEmpty(response)) { +// println('isProxyAndTryTimesToRequest 请求response为空') + //控制sleep时间 + //Thread.sleep(sleepTimes); + } else { + //控制sleep时间 + //Thread.sleep(sleepTimes); +// println('isProxyAndTryTimesToRequest 请求response响应成功, 响应内容为:' + response) + break + } + } catch (Exception e) { +// println("isProxyAndTryTimesToRequest请求发生异常, 请求url为: " + url + "请求异常信息为:" + e.getMessage()) + } finally { + //统计所有请求总次数(后期统计用) +// println("目前整个" + GLOBAL_WEBSITE_SOURCE + "抓取请求总次数count为: " + count.incrementAndGet()) + _tryTimes_++ + } + } + responseMap.put("tryTimes", _tryTimes_) + responseMap.put("url", url) + return responseMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index d1d4ada4c..59ad4ae49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20211101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20211201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20211201", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220101", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0