+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211118, 20211219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+6
-5
@@ -11,7 +11,7 @@ import java.sql.ResultSet
|
||||
|
||||
class ExportOracleData {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
@@ -24,11 +24,11 @@ class ExportOracleData {
|
||||
}
|
||||
|
||||
def run() {
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def curMonth = 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
// exportGuidedPrice()
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")
|
||||
// exportCommunityGuidedPrice(lastMonth.toString()+"28")
|
||||
//
|
||||
// //exportThirdPartyPrice()//废弃掉,老表
|
||||
// exportThirdPartyPrice2(lastMonth)
|
||||
@@ -47,12 +47,13 @@ class ExportOracleData {
|
||||
|
||||
// exportAvmEvaluationGJNB(curMonth)
|
||||
// exportAVMEvaluationItemGJNB(curMonth)
|
||||
// exportCommunityBasePrice(curMonth)
|
||||
|
||||
//
|
||||
// exportCityPrice(curMonth)
|
||||
// exportRegionPrice(curMonth)
|
||||
|
||||
// exportCommunityBasePriceGj(curMonth)
|
||||
// exportCommunityBasePriceGj(curMonth)//已合下面的合并
|
||||
exportCommunityBasePrice(curMonth)
|
||||
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
|
||||
+65
-23
@@ -27,8 +27,8 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.*;
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -271,7 +271,7 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
//},0)
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充数据删除3658_1228.xlsx",{d->
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
@@ -284,26 +284,26 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
// println index
|
||||
// }
|
||||
//},0)
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||
if(d['A']){
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if(newRegion) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dist._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def region = d['B']
|
||||
// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
// if(newRegion) {
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// if(dist){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def citys = CcbCommonUtils.citys
|
||||
//def rets = []
|
||||
@@ -488,6 +488,48 @@ ExcelUtils.modify("D:\\
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d ->
|
||||
// if(d['A']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// if(d['K']){
|
||||
// def formatAddress = AddressUtil.formatAddress(d['B'], d['K'])
|
||||
// def address = null
|
||||
// if(formatAddress && formatAddress.endsWith("号")) {
|
||||
// address = formatAddress
|
||||
// }
|
||||
//
|
||||
//
|
||||
// def data = new BasicDBObject();
|
||||
// data.put("m_date", new Date())
|
||||
// if(d['M']){
|
||||
// long glat = d['M'].toString().toDouble() * 10000000000L
|
||||
// long glng = d['L'].toString().toDouble() * 10000000000L
|
||||
// data.put("lat",glat)
|
||||
// data.put("lng",glng)
|
||||
//
|
||||
// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
|
||||
// if(baidu){
|
||||
// long blng = baidu.x * 10000000000L
|
||||
// long blat = baidu.y * 10000000000L
|
||||
// data.put("lat2",blat)
|
||||
// data.put("lng2",blng)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// if(address){
|
||||
// data.put("address", address)
|
||||
// }
|
||||
// index++
|
||||
// println index +" "+d['B'] + " "+data
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
def findDealCount(sqlInstance, def startDate, def endDate){
|
||||
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') group by cityname";
|
||||
|
||||
+8
-8
@@ -28,7 +28,7 @@ class TradeStatLog {
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
@@ -39,7 +39,7 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
@@ -73,13 +73,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-09-01"
|
||||
def date = "2021-10-01"
|
||||
def endDate = "2021-11-01"
|
||||
def newDate = "2021-12-01"
|
||||
def startDate = "2021-10-01"
|
||||
def date = "2021-11-01"
|
||||
def endDate = "2021-12-01"
|
||||
def newDate = "2022-01-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -129,7 +129,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\11月成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\12月成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
+88
-1
@@ -15,7 +15,94 @@ class CcbDistrictToDB {
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDistrictToDB();
|
||||
task.run()
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
def run1(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['C']
|
||||
def alias = []
|
||||
if(d['F']){
|
||||
if(name != d['F']){
|
||||
alias << d['F']
|
||||
}
|
||||
}
|
||||
|
||||
def formatAddress = null
|
||||
if(d['G']){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], d['G'])
|
||||
}
|
||||
def address = null
|
||||
if(d['D']){
|
||||
address = d['D']
|
||||
}
|
||||
if(d['E']){
|
||||
address = address + d['E']
|
||||
}
|
||||
|
||||
if(!formatAddress && address){
|
||||
formatAddress = AddressUtil.formatAddress(d['A'], address)
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['H']){
|
||||
glat = d['I'].toString().toDouble() * 10000000000L
|
||||
glng = d['H'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['H'].toString().toDouble(), d['I'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", d['A'])
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
if(alias){
|
||||
data.put("alias",alias)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
}
|
||||
}
|
||||
}
|
||||
// }
|
||||
},0);
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+96
-23
@@ -15,7 +15,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K']) {
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']);
|
||||
@@ -32,7 +32,7 @@ DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price"
|
||||
//});
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K']){
|
||||
//
|
||||
@@ -63,7 +63,7 @@ DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price"
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K'] && d['G']){
|
||||
//
|
||||
@@ -303,27 +303,100 @@ def city = "
|
||||
//});
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
if(d['K'] && d['K']=="别名(高德)" && d['G']){
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
// if(d['K'] && d['K']=="别名(高德)" && d['G']){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
|
||||
// def dist = null
|
||||
// if(rets){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['C'])){
|
||||
// alias << d['C']
|
||||
// }
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//});
|
||||
|
||||
|
||||
|
||||
//def rets = []
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d ->
|
||||
// if(d['A']){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['E'])){
|
||||
// alias << d['E']
|
||||
// }
|
||||
// if(alias && dist.mapping_alias!=alias){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// rets << [dist._id.toString(), d['E']]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据")
|
||||
|
||||
|
||||
def rets = []
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d ->
|
||||
// if(d['A']){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// def dists = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
|
||||
// def dist = null
|
||||
// if(dists && dists.size()==1){
|
||||
// dist = dists[0]
|
||||
// }
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['D'])){
|
||||
// alias << d['D']
|
||||
// }
|
||||
// if(alias && dist.mapping_alias!=alias){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// rets << [dist._id.toString(), d['D']]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据")
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d ->
|
||||
if(d['A']) {
|
||||
def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
if(newRegion){
|
||||
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
|
||||
def dist = null
|
||||
if(rets){
|
||||
dist = rets[0]
|
||||
}
|
||||
if(dist){
|
||||
def alias = dist.mapping_alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(d['C'])){
|
||||
alias << d['C']
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mapping_alias",alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
if (newRegion) {
|
||||
def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
rets << [dist._id.toString(), d['C']]
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
})
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据")
|
||||
|
||||
+2
-2
@@ -48,7 +48,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-11-01"
|
||||
static def statDate = "2021-12-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年12月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+6
-3
@@ -244,9 +244,12 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
def avg_charge = rets.sum()/rets.size();
|
||||
avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge)
|
||||
return avg_charge;
|
||||
if(rets){
|
||||
def avg_charge = rets.sum()/rets.size();
|
||||
avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge)
|
||||
return avg_charge;
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getPriceInRegionRank(d, nbLists, priceMap){
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
public class ChromeDriverPool {
|
||||
|
||||
private static ConcurrentHashMap<Integer,Object> pool ;//线程池
|
||||
private static int coreSize = 2;//核心线程池,一直驻留的资源
|
||||
private static int maxSize = 10;//TODO 设置最大数量资源,用于解决瞬时并发,瞬时并发过后要定时去识别空闲的释放掉,保留coreSize的资源
|
||||
private static AtomicInteger count = new AtomicInteger(0);//原子自增!!
|
||||
|
||||
static {
|
||||
//驱动初始化
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
pool = new ConcurrentHashMap<>();
|
||||
}
|
||||
|
||||
private ChromeDriverPool(){
|
||||
}
|
||||
|
||||
public static void init(int coreSize){//线程池数量初始化
|
||||
ChromeDriverPool.coreSize = coreSize;
|
||||
}
|
||||
|
||||
public static Worker getPool(){
|
||||
if(count.getAndIncrement() < coreSize){//自增!!!
|
||||
return push(new Worker());
|
||||
}else{
|
||||
return pop();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static Worker push(Worker worker){
|
||||
pool.put(count.get(),worker);
|
||||
return worker;
|
||||
}
|
||||
|
||||
/**
|
||||
* 循环获取未使用的资源
|
||||
* @return
|
||||
*/
|
||||
public static Worker pop(){
|
||||
Set<Map.Entry<Integer, Object>> sets = pool.entrySet();
|
||||
Iterator<Map.Entry<Integer, Object>> its = sets.iterator();
|
||||
while(its.hasNext()){
|
||||
Map.Entry<Integer, Object> workEntry = its.next();
|
||||
Worker work = (Worker)workEntry.getValue();
|
||||
if(work.status == 1){
|
||||
return work;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+428
@@ -0,0 +1,428 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.gj.paimai.ChromeDriverPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
|
||||
public class LianjiaCrawlByChrome {
|
||||
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityUrls = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
|
||||
for(def cityUrl : cityUrls){
|
||||
def detailUrls = getPageUrls(driver, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
poolCrawl( detailUrls)
|
||||
}
|
||||
}
|
||||
|
||||
def poolCrawl(detailUrls){
|
||||
//模拟并发
|
||||
int parallelSize = 10;
|
||||
final CountDownLatch latch = new CountDownLatch(1);
|
||||
ExecutorService pool = Executors.newFixedThreadPool(parallelSize);
|
||||
|
||||
for (int i = 0; i < detailUrls.size(); i++) {
|
||||
final def detailUrl = detailUrls[i]
|
||||
pool.execute(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
|
||||
Worker work = null;
|
||||
try {
|
||||
latch.await();//阻塞所有线程
|
||||
System.out.println(Thread.currentThread().getName());
|
||||
while (true) {//业务需求,一直等待直到获取到驱动
|
||||
work = ChromeDriverPool.getPool();
|
||||
if (work != null) {
|
||||
break;
|
||||
}
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
work.start();
|
||||
ChromeDriver driver = work.getDriver();
|
||||
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
|
||||
println detail.city + " " + detail.region + " " + detail.district_name
|
||||
System.out.println(Thread.currentThread().getName()+":"+driver);
|
||||
Thread.sleep(1000 * 10);
|
||||
} catch (Exception e) {
|
||||
System.out.println(e.getMessage());
|
||||
}finally {
|
||||
work.shutdown();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.countDown();
|
||||
}
|
||||
|
||||
def run(){
|
||||
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
// System.setProperty("http.nonProxyHosts", "localhost");
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
// def cityMap = getAllCityUrls(driver)
|
||||
// getLogin(driver)
|
||||
// crawlData(driver, cityMap)
|
||||
|
||||
addDealRegion();
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
keys << ["city":d.city, "name":d.district_name]
|
||||
}
|
||||
for(def m : keys){
|
||||
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
|
||||
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
|
||||
if(list && list.size()==1){
|
||||
def dist = list[0]
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", m.city)
|
||||
query.put("district_name", m.name)
|
||||
query.put("source", "链家在线")
|
||||
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
def count = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
|
||||
def getPageUrls(driver, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findRegionByThirdPary(city, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("name", district_name)
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
|
||||
def dbo = null;
|
||||
for(def d : dbcursor){
|
||||
dbo = d
|
||||
}
|
||||
if(dbo){
|
||||
return dbo.region
|
||||
}
|
||||
return null;
|
||||
}
|
||||
def getContent(driver, detailUrl){
|
||||
driver.get(detailUrl)
|
||||
def str = driver.getPageSource()
|
||||
if(str && str.contains("<title>登录</title>")){
|
||||
getLogin(driver)
|
||||
return getContent(driver, detailUrl)
|
||||
}
|
||||
return str;
|
||||
}
|
||||
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
|
||||
if(detailUrls){
|
||||
for(def i=0;i<detailUrls.size();i++){
|
||||
def detailUrl = detailUrls[i]
|
||||
try{
|
||||
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
|
||||
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
|
||||
def content = getContent(driver, detailUrl)
|
||||
def doc = Jsoup.parse(content);
|
||||
def eles = JQueryUtils.select(".deal-bread a", doc)
|
||||
def detail = new BasicDBObject()
|
||||
if(eles && eles.size()>=4){
|
||||
detail.city = eles[0].text().replace("房产网","")
|
||||
detail.region = eles[2].text().replace("二手房成交","")
|
||||
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
|
||||
}
|
||||
if(detail.district_name) {
|
||||
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(DEAL_PRICE_COLL, detail)
|
||||
if (!dbo) {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
|
||||
}
|
||||
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
|
||||
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
|
||||
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = detailUrl
|
||||
detail.c_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
q.put("region", detail.region)
|
||||
q.put("district_name", detail.district_name)
|
||||
q.put("area", detail.area)
|
||||
q.put("total_price", detail.total_price)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def getAllCityUrls(driver){
|
||||
driver.get("https://www.lianjia.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(!href.contains("fang.lianjia")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class Worker {
|
||||
private ChromeDriver driver;
|
||||
/**
|
||||
* 状态修改要让其他线程可见
|
||||
* 1就绪 2运行
|
||||
*/
|
||||
private volatile int status;
|
||||
|
||||
public Worker(){
|
||||
driver = new ChromeDriver();
|
||||
status = 1;
|
||||
}
|
||||
|
||||
|
||||
public void shutdown(){
|
||||
this.status = 1;
|
||||
}
|
||||
|
||||
|
||||
public void start(){
|
||||
this.status = 2;
|
||||
}
|
||||
|
||||
public ChromeDriver getDriver() {
|
||||
return driver;
|
||||
}
|
||||
}
|
||||
+28
-26
@@ -42,35 +42,38 @@ class Esf58Crawl extends BaseBatch{
|
||||
//"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
|
||||
// "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362",
|
||||
// "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222",
|
||||
// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","三亚":"sanya|2422","阜阳":"fy|2325",
|
||||
// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
|
||||
// "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
|
||||
// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385",
|
||||
def cityList = ["北京":"bj|1","上海":"sh|2","深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606",
|
||||
"宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572",
|
||||
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
|
||||
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
|
||||
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
|
||||
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
|
||||
"莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761",
|
||||
// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2",
|
||||
// "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
|
||||
// "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
|
||||
// "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
|
||||
// "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
|
||||
// "淮北":"huaibei|9357","滁州":"chuzhou|10266",
|
||||
// "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
|
||||
// "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
|
||||
// "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
|
||||
// "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752",
|
||||
def cityList = ["武夷山":"wuyishan|10761",
|
||||
"石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757",
|
||||
"福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133",
|
||||
"漳浦":"zhangpu|717", "珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791",
|
||||
"肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303",
|
||||
"漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629",
|
||||
"茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389",
|
||||
"阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449",
|
||||
"潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725",
|
||||
"博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456",
|
||||
"南宁":"nn|845","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046",
|
||||
"桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046",
|
||||
"北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340",
|
||||
"来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774",
|
||||
"北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119",
|
||||
"遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506",
|
||||
"黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506",
|
||||
"毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628",
|
||||
"清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154",
|
||||
"酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428",
|
||||
"陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390",
|
||||
"海口":"haikou|2053","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
|
||||
"五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136",
|
||||
"文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184",
|
||||
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380",
|
||||
"东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380",
|
||||
"儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977",
|
||||
"平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342",
|
||||
"濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347",
|
||||
@@ -82,7 +85,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
"舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322",
|
||||
"沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185",
|
||||
"范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603",
|
||||
"哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
|
||||
"大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718",
|
||||
"佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862",
|
||||
"伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721",
|
||||
"肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032",
|
||||
@@ -93,21 +96,21 @@ class Esf58Crawl extends BaseBatch{
|
||||
"广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897",
|
||||
"南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436",
|
||||
"当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660",
|
||||
"长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914",
|
||||
"湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
|
||||
"长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914",
|
||||
"湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756",
|
||||
"永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091",
|
||||
"澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921",
|
||||
"祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473",
|
||||
"邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532",
|
||||
"湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095",
|
||||
"保定":"bd|424","廊坊":"lf|772","秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993",
|
||||
"邢台":"xt|751","衡水":"hs|993",
|
||||
"承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048",
|
||||
"正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577",
|
||||
"雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657",
|
||||
"沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395",
|
||||
"固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066",
|
||||
"沙河":"shaheshi|755", "徐州":"xz|471", "扬州":"yz|637","淮安":"ha|968",
|
||||
"泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279",
|
||||
"沙河":"shaheshi|755",
|
||||
"宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279",
|
||||
"如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147",
|
||||
"扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402",
|
||||
"邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401",
|
||||
@@ -123,7 +126,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
"白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171",
|
||||
"梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708",
|
||||
"梨树县":"lishu|10176",
|
||||
"鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722",
|
||||
"鞍山":"as|523","抚顺":"fushun|5722",
|
||||
"营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038",
|
||||
"本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306",
|
||||
"瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733",
|
||||
@@ -131,10 +134,10 @@ class Esf58Crawl extends BaseBatch{
|
||||
"包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015",
|
||||
"呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976",
|
||||
"乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043",
|
||||
"西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909",
|
||||
"海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909",
|
||||
"黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904",
|
||||
"泰安":"ta|686","聊城":"lc|882",
|
||||
"枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","菏泽":"heze|5632",
|
||||
"枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632",
|
||||
"滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146",
|
||||
"寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690",
|
||||
"高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235",
|
||||
@@ -170,8 +173,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
"丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409",
|
||||
"德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432",
|
||||
"怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533",
|
||||
"宁波":"nb|135","金华":"jh|531",
|
||||
"绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793",
|
||||
"丽水":"lishui|7921","衢州":"quzhou|6793",
|
||||
"舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333",
|
||||
"诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334",
|
||||
"长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536",
|
||||
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.trade
|
||||
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.http.Header
|
||||
import org.apache.http.HttpResponse
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
import org.apache.http.client.HttpClient
|
||||
|
||||
class LianJiaTradeCrawl {
|
||||
|
||||
def static final LOGIN_URL = "https://clogin.lianjia.com/authentication/authenticate"
|
||||
|
||||
public def CHARSET = "utf-8";
|
||||
public def ERROR_MSG = "访问过于频繁";
|
||||
|
||||
static main(args) {
|
||||
def task = new LianJiaTradeCrawl()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def valueMap = getUidAndSid();
|
||||
def xqMap = goToXiaoqu(valueMap);
|
||||
// goLogin()
|
||||
}
|
||||
|
||||
def getUidAndSid(){
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("https://sh.lianjia.com/");
|
||||
get.setHeader("Accept-Encoding", "gzip, deflate, br")
|
||||
get.setHeader("Accept-Language", "zh-CN,zh;q=0.9")
|
||||
get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
|
||||
HttpResponse response = client.execute(get);
|
||||
def valueMap = getUidAndSid(response)
|
||||
return valueMap;
|
||||
|
||||
}
|
||||
|
||||
def goToXiaoqu(valueMap){
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("https://sh.lianjia.com/xiaoqu/");
|
||||
get.setHeader("Host","sh.lianjia.com")
|
||||
get.setHeader("Connection","keep-alive")
|
||||
get.setHeader("sec-ch-ua-mobile","?0")
|
||||
get.setHeader("sec-ch-ua-platform","Windows")
|
||||
get.setHeader("Upgrade-Insecure-Requests","1")
|
||||
get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
|
||||
get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
get.setHeader("Sec-Fetch-Site","same-origin")
|
||||
get.setHeader("Sec-Fetch-Mode","navigate")
|
||||
get.setHeader("Sec-Fetch-User","?1")
|
||||
get.setHeader("Sec-Fetch-Dest","document")
|
||||
get.setHeader("Referer","https://sh.lianjia.com/")
|
||||
get.setHeader("Accept-Encoding","gzip, deflate, br")
|
||||
get.setHeader("Accept-Language","zh-CN,zh;q=0.9")
|
||||
|
||||
def cookieStr = ""
|
||||
valueMap.each {k,v->
|
||||
cookieStr = cookieStr +k +"="+v+";"
|
||||
}
|
||||
get.setHeader("Cookie", cookieStr)
|
||||
|
||||
HttpResponse response = client.execute(get);
|
||||
def newVvalueMap = getUidAndSid(response)
|
||||
return valueMap;
|
||||
}
|
||||
|
||||
|
||||
def getUidAndSid(HttpResponse response) {
|
||||
Header[] headers = response.getHeaders("Set-Cookie");
|
||||
def valueMap = [:]
|
||||
for(def header : headers){
|
||||
String[] split = header.getValue().split(";");
|
||||
for (String s : split) {
|
||||
def arrs = s.split("=")
|
||||
valueMap.put(arrs[0], arrs[1])
|
||||
}
|
||||
}
|
||||
return valueMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def goLogin() {
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","clogin.lianjia.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","1415")
|
||||
headers.put("sec-ch-ua",'Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"')
|
||||
headers.put("sec-ch-ua-mobile","?0")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
|
||||
headers.put("sec-ch-ua-platform","Windows")
|
||||
headers.put("Content-Type","application/json")
|
||||
headers.put("Accept","*/*")
|
||||
headers.put("Sec-Fetch-Site","same-origin")
|
||||
headers.put("Sec-Fetch-Mode","cors")
|
||||
headers.put("Sec-Fetch-Dest","empty")
|
||||
headers.put("Cookie","Cookie: lianjia_uuid=45384ffe-4245-46fb-aa3f-aa6931a0a392; lianjia_ssid=75aa3a46-9d5f-42a9-b39b-4fc5e181e37c; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22%24device_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; select_city=350100; crosSdkDT2019DeviceId=-y4aedm--2s8wtc-85pqx15zlsvkewy-gkpcb9u2t")
|
||||
|
||||
headers.put("Accept-Encoding", "gzip, deflate, br")
|
||||
headers.put("Accept-Language", "zh-CN,zh;q=0.9")
|
||||
// headers.put("User-Agent", UserAgents.get())
|
||||
headers.put("Referer", "https://clogin.lianjia.com/authentication/sdk/init?version=2.0")
|
||||
headers.put("Origin", "https://clogin.lianjia.com")
|
||||
//params信息
|
||||
Map<String, String> params = new LinkedHashMap<String, String>()
|
||||
params.put("mainAuthMethodName", "username-password")
|
||||
params.put("srcId", "eyJ0Ijoie1wiZGF0YVwiOlwiMWM1ZDg5ZTFjOTc0ZWViODliOTJkYTE1ZWEzOWM5ZDY1MWNlODJkMjkzYzQzYmY2NjRiMWEyYWI2NDRkYmViY2RiMDVkNmY3YTc3MmVkODBlZjNhMjBjYWM5ODVlNjY0ODc3YmVjMTBjYTRkM2RkNWFhOTY0Mjc0OTliNzVkZWU4ZDU1YzVkNzA1ZTRhMzcyY2U1MGZkMGVmZDE0OWY0NzYxMzk4MjRlMjU0Yzk0MzY0MDJjMjdkYjkxODdjNDliOWU4MWJlNWJjNGY2ZmEyNmY5Yjk2NzAxOTFkZDQwOWMxMzhjNTEwOTUwMmQ1OGE4NTE0Y2IzMzgxNzQ0ZTgwMzg2NmVjOGZhOWIyNjc1MzEyMTQ5YjE2NjU3ZjEyNDYyY2RiOTc4ZjM2ODVlNWVmYWViMGU1YjEwZmFiMWUzNDFhMzg4NmYzZmMwM2FhZWNjZDg1ZmIwNGYzZDQ2NmRjNlwiLFwia2V5X2lkXCI6XCIxXCIsXCJzaWduXCI6XCIzYjYwYTYwOFwifSIsInIiOiJodHRwczovL2Nsb2dpbi5rZS5jb20vbG9naW4/c2VydmljZT1odHRwcyUzQSUyRiUyRnd3dy5rZS5jb20lMkZ1c2VyJTJGY2hlY2tsb2dpbiUzRnJlZGlyZWN0JTNEaHR0cHMlMjUzQSUyNTJGJTI1MkZmei5rZS5jb20lMjUyRmNoZW5namlhbyUyNTJGYzM0MjAwMzI1MDU1MzAxMDElMjUyRiIsIm9zIjoid2ViIiwidiI6IjAuMSJ9")
|
||||
params.put("version", "2.0")
|
||||
params.put("accountSystem", "customer")
|
||||
params.put("service", "https://www.lianjia.com/user/checklogin?redirect=https%3A%2F%2Ffz.lianjia.com%2Fchengjiao%2F")
|
||||
params.put("loginTicketId", "nM7eXE8ZWBP67aiapHqZRpn0QHTfCX4q")
|
||||
|
||||
|
||||
Map<String, String> credentialparams = new LinkedHashMap<String, String>()
|
||||
credentialparams.put("username", "15021384995")
|
||||
credentialparams.put("password", "dx/vjEHyEc8ETOqKDn5ZhNQZuO5JjoCu1D0UXVTQ2UrOE/liQYeNcj9H9bPB5c43UDRyKZSvA9UzBc4ycTOkDkq7pgITkJrzP0Iagkj19R0TT/yX/9NHEwm/GaqBfVvfJXfmw6ZX9vCYLaOaiOJ31PTpI5hDUS/PxxHBkmqC7Bg=")
|
||||
credentialparams.put("ioaRunState", false)
|
||||
credentialparams.put("ioaMid", "{}")
|
||||
credentialparams.put("alertSt", "")
|
||||
credentialparams.put("ioaStateVersion", 2)
|
||||
credentialparams.put("encodeVersion", "1")
|
||||
params.put("credential", credentialparams)
|
||||
|
||||
Map<String, String> contextparams = new LinkedHashMap<String, String>()
|
||||
contextparams.put("deviceId", "default")
|
||||
contextparams.put("sign", "default")
|
||||
params.put("context", contextparams)
|
||||
|
||||
//post请求
|
||||
responseMap = isProxyAndTryTimesToRequest(LOGIN_URL, HttpClient.CHARSET, 100, "POST", params, headers, true)
|
||||
} catch (Exception e) {
|
||||
// println "goLogin(${username}, ${password}, ${rememberme})发生异常, 异常信息为:" + e.getMessage()
|
||||
}
|
||||
return responseMap
|
||||
}
|
||||
|
||||
|
||||
def static isProxyAndTryTimesToRequest(url, charset, tryTimes, requestType, params, headers, isJson) {
|
||||
def responseMap = [:]
|
||||
def _tryTimes_ = 0
|
||||
//请求响应为空(切换代理重试)
|
||||
while (_tryTimes_ < tryTimes) {
|
||||
try {
|
||||
//第一次请求不提示重试
|
||||
// println("请求地址URL为:" + url + "代理尝试次数tryTimes为" + (_tryTimes_ + 1))
|
||||
if (StringUtils.isNotEmpty(requestType) && requestType.equals("POST")) {
|
||||
responseMap = ABuYunUtil.doPostResponse(url, params, headers, charset, isJson)
|
||||
} else {
|
||||
responseMap = ABuYunUtil.doGetResponse(url, params, headers, charset)
|
||||
}
|
||||
def response = responseMap.get("response")
|
||||
if (StringUtils.isEmpty(response)) {
|
||||
// println('isProxyAndTryTimesToRequest 请求response为空')
|
||||
//控制sleep时间
|
||||
//Thread.sleep(sleepTimes);
|
||||
} else {
|
||||
//控制sleep时间
|
||||
//Thread.sleep(sleepTimes);
|
||||
// println('isProxyAndTryTimesToRequest 请求response响应成功, 响应内容为:' + response)
|
||||
break
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// println("isProxyAndTryTimesToRequest请求发生异常, 请求url为: " + url + "请求异常信息为:" + e.getMessage())
|
||||
} finally {
|
||||
//统计所有请求总次数(后期统计用)
|
||||
// println("目前整个" + GLOBAL_WEBSITE_SOURCE + "抓取请求总次数count为: " + count.incrementAndGet())
|
||||
_tryTimes_++
|
||||
}
|
||||
}
|
||||
responseMap.put("tryTimes", _tryTimes_)
|
||||
responseMap.put("url", url)
|
||||
return responseMap
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20211101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20211201", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20211201", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220101", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
Reference in New Issue
Block a user