修改房源时间

This commit is contained in:
2022-01-19 14:30:39 +08:00
parent 2a37912e7e
commit 9100314a96
14 changed files with 1007 additions and 94 deletions
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211118, 20211219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -11,7 +11,7 @@ import java.sql.ResultSet
class ExportOracleData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -24,11 +24,11 @@ class ExportOracleData {
}
def run() {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def curMonth = 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// exportGuidedPrice()
exportCommunityGuidedPrice(lastMonth.toString()+"28")
// exportCommunityGuidedPrice(lastMonth.toString()+"28")
//
// //exportThirdPartyPrice()//废弃掉,老表
// exportThirdPartyPrice2(lastMonth)
@@ -47,12 +47,13 @@ class ExportOracleData {
// exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth)
// exportCommunityBasePrice(curMonth)
//
// exportCityPrice(curMonth)
// exportRegionPrice(curMonth)
// exportCommunityBasePriceGj(curMonth)
// exportCommunityBasePriceGj(curMonth)//已合下面的合并
exportCommunityBasePrice(curMonth)
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
@@ -27,8 +27,8 @@ import java.util.concurrent.atomic.AtomicLong
import java.util.regex.*;
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -271,7 +271,7 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
//},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充数据删除3658_1228.xlsx",{d->
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d->
// if(d['A']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
@@ -284,26 +284,26 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
// println index
// }
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
if(d['A']){
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if(newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dist._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d
}
}
}
},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
// if(d['A']){
// def region = d['B']
// def newRegion = RegionData.formatRegion(d['A'], region)
// if(newRegion) {
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
// if(dist){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d
// }
// }
// }
//},0)
//def citys = CcbCommonUtils.citys
//def rets = []
@@ -488,6 +488,48 @@ ExcelUtils.modify("D:\\
// }
//},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d ->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// if(d['K']){
// def formatAddress = AddressUtil.formatAddress(d['B'], d['K'])
// def address = null
// if(formatAddress && formatAddress.endsWith("号")) {
// address = formatAddress
// }
//
//
// def data = new BasicDBObject();
// data.put("m_date", new Date())
// if(d['M']){
// long glat = d['M'].toString().toDouble() * 10000000000L
// long glng = d['L'].toString().toDouble() * 10000000000L
// data.put("lat",glat)
// data.put("lng",glng)
//
// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
// if(baidu){
// long blng = baidu.x * 10000000000L
// long blat = baidu.y * 10000000000L
// data.put("lat2",blat)
// data.put("lng2",blng)
// }
//
// }
//
// if(address){
// data.put("address", address)
// }
// index++
// println index +" "+d['B'] + " "+data
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data))
// }
// }
// }
//})
def findDealCount(sqlInstance, def startDate, def endDate){
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') group by cityname";
@@ -28,7 +28,7 @@ class TradeStatLog {
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -39,7 +39,7 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
@@ -73,13 +73,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-09-01"
def date = "2021-10-01"
def endDate = "2021-11-01"
def newDate = "2021-12-01"
def startDate = "2021-10-01"
def date = "2021-11-01"
def endDate = "2021-12-01"
def newDate = "2022-01-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -129,7 +129,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\11月成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\12月成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -15,7 +15,94 @@ class CcbDistrictToDB {
static main(args) {
def task = new CcbDistrictToDB();
task.run()
// task.run()
task.run1()
}
def run1(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
def city = d['A']
def region = d['B']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['C']
def alias = []
if(d['F']){
if(name != d['F']){
alias << d['F']
}
}
def formatAddress = null
if(d['G']){
formatAddress = AddressUtil.formatAddress(d['A'], d['G'])
}
def address = null
if(d['D']){
address = d['D']
}
if(d['E']){
address = address + d['E']
}
if(!formatAddress && address){
formatAddress = AddressUtil.formatAddress(d['A'], address)
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['H']){
glat = d['I'].toString().toDouble() * 10000000000L
glng = d['H'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['H'].toString().toDouble(), d['I'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}
if(name){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", d['A'])
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
if(alias){
data.put("alias",alias)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
}
}
}
// }
},0);
}
def run(){
@@ -15,7 +15,7 @@ import com.mongodb.DBCollection
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K']) {
// def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']);
@@ -32,7 +32,7 @@ DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price"
//});
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K']){
//
@@ -63,7 +63,7 @@ DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price"
//
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K'] && d['G']){
//
@@ -303,27 +303,100 @@ def city = "
//});
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
if(d['K'] && d['K']=="别名(高德)" && d['G']){
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
// if(d['K'] && d['K']=="别名(高德)" && d['G']){
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion){
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
// def dist = null
// if(rets){
// dist = rets[0]
// }
// if(dist){
// def alias = dist.mapping_alias
// if(!alias){
// alias = []
// }
// if(!alias.contains(d['C'])){
// alias << d['C']
// }
// def udata = new BasicDBObject()
// udata.put("mapping_alias",alias)
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// }
// }
// }
//});
//def rets = []
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d ->
// if(d['A']){
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion){
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
// if(dist){
// def alias = dist.mapping_alias
// if(!alias){
// alias = []
// }
// if(!alias.contains(d['E'])){
// alias << d['E']
// }
// if(alias && dist.mapping_alias!=alias){
// def udata = new BasicDBObject()
// udata.put("mapping_alias",alias)
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// }
// rets << [dist._id.toString(), d['E']]
// }
// }
// }
//})
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据")
def rets = []
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d ->
// if(d['A']){
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion){
// def dists = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
// def dist = null
// if(dists && dists.size()==1){
// dist = dists[0]
// }
// if(dist){
// def alias = dist.mapping_alias
// if(!alias){
// alias = []
// }
// if(!alias.contains(d['D'])){
// alias << d['D']
// }
// if(alias && dist.mapping_alias!=alias){
// def udata = new BasicDBObject()
// udata.put("mapping_alias",alias)
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// }
// rets << [dist._id.toString(), d['D']]
// }
// }
// }
//})
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据")
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d ->
if(d['A']) {
def newRegion = RegionData.formatRegion(d['A'], d['B'])
if(newRegion){
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
def dist = null
if(rets){
dist = rets[0]
}
if(dist){
def alias = dist.mapping_alias
if(!alias){
alias = []
}
if(!alias.contains(d['C'])){
alias << d['C']
}
def udata = new BasicDBObject()
udata.put("mapping_alias",alias)
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
if (newRegion) {
def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
if (dist) {
rets << [dist._id.toString(), d['C']]
}
}
}
});
})
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据")
@@ -48,7 +48,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-11-01"
static def statDate = "2021-12-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年12月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -244,9 +244,12 @@ class DistrictIndexBatch extends BaseBatch{
}
}
}
def avg_charge = rets.sum()/rets.size();
avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge)
return avg_charge;
if(rets){
def avg_charge = rets.sum()/rets.size();
avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge)
return avg_charge;
}
return null
}
def getPriceInRegionRank(d, nbLists, priceMap){
@@ -0,0 +1,61 @@
package com.fangjia.script.gj.paimai
import org.openqa.selenium.chrome.ChromeDriver
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicInteger
public class ChromeDriverPool {
private static ConcurrentHashMap<Integer,Object> pool ;//线程池
private static int coreSize = 2;//核心线程池,一直驻留的资源
private static int maxSize = 10;//TODO 设置最大数量资源,用于解决瞬时并发,瞬时并发过后要定时去识别空闲的释放掉,保留coreSize的资源
private static AtomicInteger count = new AtomicInteger(0);//原子自增!!
static {
//驱动初始化
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
pool = new ConcurrentHashMap<>();
}
private ChromeDriverPool(){
}
public static void init(int coreSize){//线程池数量初始化
ChromeDriverPool.coreSize = coreSize;
}
public static Worker getPool(){
if(count.getAndIncrement() < coreSize){//自增!!!
return push(new Worker());
}else{
return pop();
}
}
public static Worker push(Worker worker){
pool.put(count.get(),worker);
return worker;
}
/**
* 循环获取未使用的资源
* @return
*/
public static Worker pop(){
Set<Map.Entry<Integer, Object>> sets = pool.entrySet();
Iterator<Map.Entry<Integer, Object>> its = sets.iterator();
while(its.hasNext()){
Map.Entry<Integer, Object> workEntry = its.next();
Worker work = (Worker)workEntry.getValue();
if(work.status == 1){
return work;
}
}
return null;
}
}
@@ -0,0 +1,428 @@
package com.fangjia.script.gj.paimai
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.gj.paimai.ChromeDriverPool
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new LianjiaCrawlByChrome();
task.run()
// task.run1()
}
def run1(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityUrls = getAllCityUrls(driver)
getLogin(driver)
for(def cityUrl : cityUrls){
def detailUrls = getPageUrls(driver, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
poolCrawl( detailUrls)
}
}
def poolCrawl(detailUrls){
//模拟并发
int parallelSize = 10;
final CountDownLatch latch = new CountDownLatch(1);
ExecutorService pool = Executors.newFixedThreadPool(parallelSize);
for (int i = 0; i < detailUrls.size(); i++) {
final def detailUrl = detailUrls[i]
pool.execute(new Runnable() {
@Override
public void run() {
Worker work = null;
try {
latch.await();//阻塞所有线程
System.out.println(Thread.currentThread().getName());
while (true) {//业务需求,一直等待直到获取到驱动
work = ChromeDriverPool.getPool();
if (work != null) {
break;
}
Thread.sleep(1000);
}
work.start();
ChromeDriver driver = work.getDriver();
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
println detail.city + " " + detail.region + " " + detail.district_name
System.out.println(Thread.currentThread().getName()+":"+driver);
Thread.sleep(1000 * 10);
} catch (Exception e) {
System.out.println(e.getMessage());
}finally {
work.shutdown();
}
}
});
}
latch.countDown();
}
def run(){
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
// System.setProperty("http.nonProxyHosts", "localhost");
// WebDriver driver = new ChromeDriver()
// def cityMap = getAllCityUrls(driver)
// getLogin(driver)
// crawlData(driver, cityMap)
addDealRegion();
}
def addDealRegion(){
def q = new BasicDBObject()
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
for(def d : dbcursor){
keys << ["city":d.city, "name":d.district_name]
}
for(def m : keys){
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
if(list && list.size()==1){
def dist = list[0]
def query = new BasicDBObject()
query.put("city", m.city)
query.put("district_name", m.name)
query.put("source", "链家在线")
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
def data = new BasicDBObject()
data.put("region", dist.region)
def count = DEAL_PRICE_COLL.count(query)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
}
def crawlData(driver, cityMap){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
def getPageUrls(driver, cityUrl){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
stop = true;
break;
}
}
page++;
}else{
stop = true;
}
}
return detailUrls
}
def getPageDeail(driver, city, cityUrl){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
}else{
stop = true;
break;
}
}
page++;
}else{
stop = true;
}
}
return detailUrls
}
def parse(city,e){
try {
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.url = e.select(".title a").attr("href")
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}catch(Exception ex){
ex.printStackTrace()
println e.select(".title a").attr("href")
}
}
def findRegionByThirdPary(city, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", district_name)
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
def dbo = null;
for(def d : dbcursor){
dbo = d
}
if(dbo){
return dbo.region
}
return null;
}
def getContent(driver, detailUrl){
driver.get(detailUrl)
def str = driver.getPageSource()
if(str && str.contains("<title>登录</title>")){
getLogin(driver)
return getContent(driver, detailUrl)
}
return str;
}
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
if(detailUrls){
for(def i=0;i<detailUrls.size();i++){
def detailUrl = detailUrls[i]
try{
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
} catch (Exception e) {
e.printStackTrace()
}
}
}
}
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
def content = getContent(driver, detailUrl)
def doc = Jsoup.parse(content);
def eles = JQueryUtils.select(".deal-bread a", doc)
def detail = new BasicDBObject()
if(eles && eles.size()>=4){
detail.city = eles[0].text().replace("房产网","")
detail.region = eles[2].text().replace("二手房成交","")
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
}
if(detail.district_name) {
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
def dbo = findTrade(DEAL_PRICE_COLL, detail)
if (!dbo) {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.url = detailUrl
detail.c_date = new Date()
detail.m_date = new Date()
DEAL_PRICE_COLL.save(detail)
}
}
return detail;
}
def findTrade(def detail){
def q = new BasicDBObject()
q.put("city", detail.city)
q.put("region", detail.region)
q.put("district_name", detail.district_name)
q.put("area", detail.area)
q.put("total_price", detail.total_price)
q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo
}
def getAllCityUrls(driver){
driver.get("https://www.lianjia.com/city/")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
}
return cityMap
}
def getLogin(driver){
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
}
}
@@ -0,0 +1,31 @@
package com.fangjia.script.gj.paimai
import org.openqa.selenium.chrome.ChromeDriver
class Worker {
private ChromeDriver driver;
/**
* 状态修改要让其他线程可见
* 1就绪 2运行
*/
private volatile int status;
public Worker(){
driver = new ChromeDriver();
status = 1;
}
public void shutdown(){
this.status = 1;
}
public void start(){
this.status = 2;
}
public ChromeDriver getDriver() {
return driver;
}
}
@@ -42,35 +42,38 @@ class Esf58Crawl extends BaseBatch{
//"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
// "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362",
// "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222",
// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","三亚":"sanya|2422","阜阳":"fy|2325",
// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
// "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385",
def cityList = ["北京":"bj|1","上海":"sh|2","深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606",
"宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572",
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
"莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761",
// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2",
// "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
// "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
// "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
// "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
// "淮北":"huaibei|9357","滁州":"chuzhou|10266",
// "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
// "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
// "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
// "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752",
def cityList = ["武夷山":"wuyishan|10761",
"石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757",
"福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133",
"漳浦":"zhangpu|717", "珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791",
"肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303",
"漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629",
"茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389",
"阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449",
"潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725",
"博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456",
"南宁":"nn|845","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046",
"桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046",
"北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340",
"来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774",
"北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119",
"遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506",
"黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506",
"毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628",
"清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154",
"酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428",
"陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390",
"海口":"haikou|2053","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
"五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
"文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184",
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380",
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380",
"儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977",
"平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342",
"濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347",
@@ -82,7 +85,7 @@ class Esf58Crawl extends BaseBatch{
"舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322",
"沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185",
"范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603",
"哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
"大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
"佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862",
"伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721",
"肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
@@ -93,21 +96,21 @@ class Esf58Crawl extends BaseBatch{
"广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897",
"南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436",
"当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660",
"长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914",
"湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
"长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914",
"湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
"永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091",
"澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921",
"祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473",
"邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532",
"湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095",
"保定":"bd|424","廊坊":"lf|772","秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993",
"邢台":"xt|751","衡水":"hs|993",
"承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048",
"正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577",
"雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657",
"沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395",
"固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066",
"沙河":"shaheshi|755", "徐州":"xz|471", "扬州":"yz|637","淮安":"ha|968",
"泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279",
"沙河":"shaheshi|755",
"宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279",
"如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147",
"扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402",
"邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401",
@@ -123,7 +126,7 @@ class Esf58Crawl extends BaseBatch{
"白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171",
"梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708",
"梨树县":"lishu|10176",
"鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722",
"鞍山":"as|523","抚顺":"fushun|5722",
"营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038",
"本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306",
"瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733",
@@ -131,10 +134,10 @@ class Esf58Crawl extends BaseBatch{
"包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015",
"呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976",
"乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043",
"西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909",
"海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909",
"黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904",
"泰安":"ta|686","聊城":"lc|882",
"枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","菏泽":"heze|5632",
"枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632",
"滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146",
"寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690",
"高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235",
@@ -170,8 +173,7 @@ class Esf58Crawl extends BaseBatch{
"丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409",
"德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432",
"怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533",
"宁波":"nb|135","金华":"jh|531",
"绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793",
"丽水":"lishui|7921","衢州":"quzhou|6793",
"舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333",
"诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334",
"长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536",
@@ -0,0 +1,185 @@
package com.fangjia.script.thirdPriceCrawl.trade
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.HttpClient
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.http.Header
import org.apache.http.HttpResponse
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
import org.apache.http.client.HttpClient
class LianJiaTradeCrawl {
def static final LOGIN_URL = "https://clogin.lianjia.com/authentication/authenticate"
public def CHARSET = "utf-8";
public def ERROR_MSG = "访问过于频繁";
static main(args) {
def task = new LianJiaTradeCrawl()
task.run()
}
def run(){
def valueMap = getUidAndSid();
def xqMap = goToXiaoqu(valueMap);
// goLogin()
}
def getUidAndSid(){
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("https://sh.lianjia.com/");
get.setHeader("Accept-Encoding", "gzip, deflate, br")
get.setHeader("Accept-Language", "zh-CN,zh;q=0.9")
get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
HttpResponse response = client.execute(get);
def valueMap = getUidAndSid(response)
return valueMap;
}
def goToXiaoqu(valueMap){
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("https://sh.lianjia.com/xiaoqu/");
get.setHeader("Host","sh.lianjia.com")
get.setHeader("Connection","keep-alive")
get.setHeader("sec-ch-ua-mobile","?0")
get.setHeader("sec-ch-ua-platform","Windows")
get.setHeader("Upgrade-Insecure-Requests","1")
get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
get.setHeader("Sec-Fetch-Site","same-origin")
get.setHeader("Sec-Fetch-Mode","navigate")
get.setHeader("Sec-Fetch-User","?1")
get.setHeader("Sec-Fetch-Dest","document")
get.setHeader("Referer","https://sh.lianjia.com/")
get.setHeader("Accept-Encoding","gzip, deflate, br")
get.setHeader("Accept-Language","zh-CN,zh;q=0.9")
def cookieStr = ""
valueMap.each {k,v->
cookieStr = cookieStr +k +"="+v+";"
}
get.setHeader("Cookie", cookieStr)
HttpResponse response = client.execute(get);
def newVvalueMap = getUidAndSid(response)
return valueMap;
}
def getUidAndSid(HttpResponse response) {
Header[] headers = response.getHeaders("Set-Cookie");
def valueMap = [:]
for(def header : headers){
String[] split = header.getValue().split(";");
for (String s : split) {
def arrs = s.split("=")
valueMap.put(arrs[0], arrs[1])
}
}
return valueMap;
}
def goLogin() {
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","clogin.lianjia.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","1415")
headers.put("sec-ch-ua",'Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"')
headers.put("sec-ch-ua-mobile","?0")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
headers.put("sec-ch-ua-platform","Windows")
headers.put("Content-Type","application/json")
headers.put("Accept","*/*")
headers.put("Sec-Fetch-Site","same-origin")
headers.put("Sec-Fetch-Mode","cors")
headers.put("Sec-Fetch-Dest","empty")
headers.put("Cookie","Cookie: lianjia_uuid=45384ffe-4245-46fb-aa3f-aa6931a0a392; lianjia_ssid=75aa3a46-9d5f-42a9-b39b-4fc5e181e37c; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22%24device_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; select_city=350100; crosSdkDT2019DeviceId=-y4aedm--2s8wtc-85pqx15zlsvkewy-gkpcb9u2t")
headers.put("Accept-Encoding", "gzip, deflate, br")
headers.put("Accept-Language", "zh-CN,zh;q=0.9")
// headers.put("User-Agent", UserAgents.get())
headers.put("Referer", "https://clogin.lianjia.com/authentication/sdk/init?version=2.0")
headers.put("Origin", "https://clogin.lianjia.com")
//params信息
Map<String, String> params = new LinkedHashMap<String, String>()
params.put("mainAuthMethodName", "username-password")
params.put("srcId", "eyJ0Ijoie1wiZGF0YVwiOlwiMWM1ZDg5ZTFjOTc0ZWViODliOTJkYTE1ZWEzOWM5ZDY1MWNlODJkMjkzYzQzYmY2NjRiMWEyYWI2NDRkYmViY2RiMDVkNmY3YTc3MmVkODBlZjNhMjBjYWM5ODVlNjY0ODc3YmVjMTBjYTRkM2RkNWFhOTY0Mjc0OTliNzVkZWU4ZDU1YzVkNzA1ZTRhMzcyY2U1MGZkMGVmZDE0OWY0NzYxMzk4MjRlMjU0Yzk0MzY0MDJjMjdkYjkxODdjNDliOWU4MWJlNWJjNGY2ZmEyNmY5Yjk2NzAxOTFkZDQwOWMxMzhjNTEwOTUwMmQ1OGE4NTE0Y2IzMzgxNzQ0ZTgwMzg2NmVjOGZhOWIyNjc1MzEyMTQ5YjE2NjU3ZjEyNDYyY2RiOTc4ZjM2ODVlNWVmYWViMGU1YjEwZmFiMWUzNDFhMzg4NmYzZmMwM2FhZWNjZDg1ZmIwNGYzZDQ2NmRjNlwiLFwia2V5X2lkXCI6XCIxXCIsXCJzaWduXCI6XCIzYjYwYTYwOFwifSIsInIiOiJodHRwczovL2Nsb2dpbi5rZS5jb20vbG9naW4/c2VydmljZT1odHRwcyUzQSUyRiUyRnd3dy5rZS5jb20lMkZ1c2VyJTJGY2hlY2tsb2dpbiUzRnJlZGlyZWN0JTNEaHR0cHMlMjUzQSUyNTJGJTI1MkZmei5rZS5jb20lMjUyRmNoZW5namlhbyUyNTJGYzM0MjAwMzI1MDU1MzAxMDElMjUyRiIsIm9zIjoid2ViIiwidiI6IjAuMSJ9")
params.put("version", "2.0")
params.put("accountSystem", "customer")
params.put("service", "https://www.lianjia.com/user/checklogin?redirect=https%3A%2F%2Ffz.lianjia.com%2Fchengjiao%2F")
params.put("loginTicketId", "nM7eXE8ZWBP67aiapHqZRpn0QHTfCX4q")
Map<String, String> credentialparams = new LinkedHashMap<String, String>()
credentialparams.put("username", "15021384995")
credentialparams.put("password", "dx/vjEHyEc8ETOqKDn5ZhNQZuO5JjoCu1D0UXVTQ2UrOE/liQYeNcj9H9bPB5c43UDRyKZSvA9UzBc4ycTOkDkq7pgITkJrzP0Iagkj19R0TT/yX/9NHEwm/GaqBfVvfJXfmw6ZX9vCYLaOaiOJ31PTpI5hDUS/PxxHBkmqC7Bg=")
credentialparams.put("ioaRunState", false)
credentialparams.put("ioaMid", "{}")
credentialparams.put("alertSt", "")
credentialparams.put("ioaStateVersion", 2)
credentialparams.put("encodeVersion", "1")
params.put("credential", credentialparams)
Map<String, String> contextparams = new LinkedHashMap<String, String>()
contextparams.put("deviceId", "default")
contextparams.put("sign", "default")
params.put("context", contextparams)
//post请求
responseMap = isProxyAndTryTimesToRequest(LOGIN_URL, HttpClient.CHARSET, 100, "POST", params, headers, true)
} catch (Exception e) {
// println "goLogin(${username}, ${password}, ${rememberme})发生异常, 异常信息为:" + e.getMessage()
}
return responseMap
}
def static isProxyAndTryTimesToRequest(url, charset, tryTimes, requestType, params, headers, isJson) {
def responseMap = [:]
def _tryTimes_ = 0
//请求响应为空(切换代理重试)
while (_tryTimes_ < tryTimes) {
try {
//第一次请求不提示重试
// println("请求地址URL为:" + url + "代理尝试次数tryTimes为" + (_tryTimes_ + 1))
if (StringUtils.isNotEmpty(requestType) && requestType.equals("POST")) {
responseMap = ABuYunUtil.doPostResponse(url, params, headers, charset, isJson)
} else {
responseMap = ABuYunUtil.doGetResponse(url, params, headers, charset)
}
def response = responseMap.get("response")
if (StringUtils.isEmpty(response)) {
// println('isProxyAndTryTimesToRequest 请求response为空')
//控制sleep时间
//Thread.sleep(sleepTimes);
} else {
//控制sleep时间
//Thread.sleep(sleepTimes);
// println('isProxyAndTryTimesToRequest 请求response响应成功, 响应内容为:' + response)
break
}
} catch (Exception e) {
// println("isProxyAndTryTimesToRequest请求发生异常, 请求url为: " + url + "请求异常信息为:" + e.getMessage())
} finally {
//统计所有请求总次数(后期统计用)
// println("目前整个" + GLOBAL_WEBSITE_SOURCE + "抓取请求总次数count为: " + count.incrementAndGet())
_tryTimes_++
}
}
responseMap.put("tryTimes", _tryTimes_)
responseMap.put("url", url)
return responseMap
}
}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20211101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20211201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20211201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220101", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0