房源时间修改

This commit is contained in:
2022-03-21 13:59:14 +08:00
parent 403ffbb8bb
commit 24cddd20fe
14 changed files with 707 additions and 68 deletions
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220118, 20220219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -40,7 +40,7 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
@@ -49,7 +49,7 @@ class TradeStatLog {
// task.deleteDealData()
// task.deleteDealData2()
// task.statTableCount()
task.filterTrade()
// task.filterTrade()
}
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","11月总量","12月总量","1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-11-01"
def date = "2021-12-01"
def endDate = "2022-01-01"
def newDate = "2022-02-01"
def startDate = "2021-12-01"
def date = "2022-01-01"
def endDate = "2022-02-01"
def newDate = "2022-03-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-01成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -1,25 +1,123 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import groovy.sql.Sql
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", ZS_MONGODB_URL)
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
if(d['B']){
//def avg_price =CommonDatautils.queryCityPrice(d['B'])
//d['C'] = avg_price
// def q = new BasicDBObject()
// q.put("area_name",new BasicDBObject( '$regex' , d['B'].replace("州","")+/.*$/))
// def obj = COMMUNITY_CONN.findOne(q)
// if(obj){
// d['D'] = obj.area_code
// println obj.area_name +" "+obj.area_code
// }
//def obj = findDistrictPriceCount(d['B'], DateUtils.parseFromTightByPattern("2022-03-13","yyyy-MM-dd"), CITY_INFO)
// d['D'] = CommonDatautils.findAllDistrictCount("district", d['B'])
// d['E'] = findHouseCount(d['B'], 20210101, 20211231)
//
// def startDate = DateUtils.parseFromTightByPattern("2021-01-01", "yyyy-MM-dd")
// def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
// def deals = findTradesDetailById(d['B'], startDate,endDate, DEAL_PRICE_COLL)
// d['F'] = deals.size()
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
d['G'] = findCcbDistCount(cityCodeDataMap.get(d['B']), sqlInstance)
}
},0)
def findCcbDistCount(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_community_gj where cityId='${cityid}' and status=0 ";
def total = null
sqlInstance.eachRow(sql,{r->
total = r.count
})
return total
}
public static def findTradesDetailById(city, startDate, endDate, DEAL_PRICE_COLL){
def q = new BasicDBObject()
q.put("city", city)
q.put("area", new BasicDBObject('$gte', 20))
q.put("avg_price", new BasicDBObject('$gt', 0))
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
lists << d
}
}
return lists
}
def findHouseCount(def city, def startDate, def endDate){
def aliasStr = null;
def total = 0
try{
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
def findDistrictPriceCount(def city, def weekend, def CITY_INFO){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def obj = CITY_INFO.findOne(city)
return obj
}
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K']) {
@@ -475,26 +573,26 @@ def city = "
//
//})
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d ->
if(d['A']){
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def mappingAlias = dist.mapping_alias
if(mappingAlias){
if(mappingAlias.contains(d['B'])){
mappingAlias.remove(d['B'])
DBObject udata = new BasicDBObject();
udata.put("mapping_alias", mappingAlias);
udata.put("m_date", new Date());
DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata))
}else{
println d['A']+"无映射别名"
}
}else{
println d['A']+"映射别名为空"
}
}else{
println d['A']+"没找到"
}
}
})
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d ->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def mappingAlias = dist.mapping_alias
// if(mappingAlias){
// if(mappingAlias.contains(d['B'])){
// mappingAlias.remove(d['B'])
// DBObject udata = new BasicDBObject();
// udata.put("mapping_alias", mappingAlias);
// udata.put("m_date", new Date());
// DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata))
// }else{
// println d['A']+"无映射别名"
// }
// }else{
// println d['A']+"映射别名为空"
// }
// }else{
// println d['A']+"没找到"
// }
// }
//})
@@ -30,15 +30,63 @@ class MatchMapData {
static main(args) {
def task = new MatchMapData()
task.run()
// task.run()
// task.run1()
// task.run2()
// task.run3()
// task.exportHouse()
// task.addDistrictLevel()
task.run4()
}
def run4() {
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
def regionCode = null
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
regionCode = getRegionCode(newRegion)
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
d['F'] = "完全一致"
d['E'] = dist.name
d['D'] = dist.name_cat
d['G'] = dist.estate_type2
} else {
dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
if (dist) {
d['F'] = "映射别名"
d['E'] = dist.name
d['D'] = dist.name_cat
d['G'] = dist.estate_type2
} else {
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
if (rets) {
if(rets.size()>1){
d['F'] = "别名-多个"
}else{
d['F'] = "别名"
}
}
if (rets) {
dist = rets[0]
}
if (dist) {
d['F'] = "完全一致"
d['E'] = dist.name
d['D'] = dist.name_cat
d['G'] = dist.estate_type2
}
}
}
}
}
}, 0)
}
def run3(){
def files = ["合肥融合匹配_1215.xlsx"]
@@ -0,0 +1,198 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
class TradeDistrictToDB {
static main(args) {
def task = new TradeDistrictToDB();
// task.run()
task.run1()
}
def run1(){
def index = 0
def keys = []
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
if(d['D']){
def key = d['A']+"##"+d['D']+"##"+d['C']
if(!keys.contains(key)){
def formatAddress = null
if(d['G']){
formatAddress = AddressUtil.formatAddress(d['A'], d['F'])
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['G']){
glat = d['H'].toString().toDouble() * 10000000000L
glng = d['G'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def name = d['C'].replace("·","").replace("II","二")
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}
keys << key
if(name){
def dist = CommonDatautils.findDistrict(d['A'], d['D'], name, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", d['A'])
data.put("region",d['D'])
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['D'], d['C'], 4);
println index+" "+data
}
}
}
}
},0)
}
def run() {
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
def city = d['A']
def region = d['B']
def newRegion = RegionData.formatRegion(city, region)
if (newRegion) {
def map = findMapData(city, newRegion, d['C'])
if(map){
d['D'] = newRegion
d['E'] = map.name
d['F'] = map.address
d['G'] = map.lng
d['H'] = map.lat
d['I'] = map.lng2
d['J'] = map.lag2
}
}
}, 0);
}
def findMapData(def city, def region, name){
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
def map = findStandardObject2(city, region, name, null, rets);
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(city, region, name);
map = findStandardObject2(city, region, name, null, rets);
}
return map;
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -218,7 +218,6 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","玉泉路甲46号院")
DBObject keys = new BasicDBObject();
if(fields){
@@ -918,6 +917,23 @@ class CommonDatautils {
}
return ret;
}
public static def findDistrictsByCityAndName(String city, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("cat", cat);
q.put("name", name);
q.put("visible", 0);
def list = []
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
list << obj;
}
}
return list;
}
public static def findDistrict(String city, String name, String cat){
@@ -0,0 +1,278 @@
package com.fangjia.script.fuzhou
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import java.util.concurrent.CountDownLatch
class ThirdTest {
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new ThirdTest()
// task.run()
// task.run1()
task.run3()
// task.run4()
}
def run4(){
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
if (!d['C']) {
// def name = d['A'].replace("(钢混)","").replace("(非钢混)","").replace("(混合)","")
// def lists = CommonDatautils.findDistrictsByCityAndName("福州",name, "district")
// if(lists){
// for(def o : lists){
//// if(d['B'] && d['B'].toString().contains(o.address)){
// d['C'] = o.region
// d['D'] = o.name
// d['E'] = "名称去括号中的字"
//// }
// }
// }
def rMap = SeaweedDataMatchProcess.doMatch("福州", null, d['A'], d['B']);
if(rMap){
d['C'] = rMap.region
d['D'] = rMap.name
d['E'] = "匹配接口"
}
}
},0)
}
def run3() {
// ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
// if(!d['F']){
// def price = CommonDatautils.queryDistrictPrice("福州",d['C'], d['D'])
// d['F'] = price
// }
// },0)
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202202.csv"))
// def map1 = [:]
// for(def s : list1){
// def row = s.split("@\\|@")
// def key = row[2]+"##"+row[3]+"##"+row[4]
// map1.put(key, row[5])
// }
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv"))
def map1 = [:]
for(def str : list1){
def s = str.split("@\\|@")
def key = s[2]+"##"+s[3]+"##"+s[4]
def priceMap = [:]
map1.put(key, priceMap)
if(s.size()>=7 && s[6]){
priceMap.put("zhugezhaofang", s[6])
}
if(s.size()>=8 && s[7]){
priceMap.put("anjuke", s[7])
}
if(s.size()>=9 && s[8]){
priceMap.put("soufang", s[8])
}
if(s.size()>=10 && s[9]){
priceMap.put("lianjia", s[9])
}
if(s.size()>=11 && s[10]){
priceMap.put("cityhouse", s[10])
}
if(s.size()>=12 && s[11]){
priceMap.put("fangtan", s[11])
}
if(s.size()>=13 && s[12]){
priceMap.put("fangdoudou", s[12])
}
if(s.size()>=14 && s[13]){
priceMap.put("maitian", s[13])
}
try{
if(s.size()>=15 && s[14]){
priceMap.put("58tc", s[14])
}
}catch(Exception e){
e.getMessage()
}
}
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
// if (!d['F']) {
// def lists = CommonDatautils.findDistrictsByCityAndName("福州", d['A'], "district")
// if(lists){
// for(def o : lists){
// if(o.address==d['B']){
// d['C'] = o.region
// }
// }
// }
// }
def key = "福州##"+d['C']+"##"+d['D']
def priceMap = map1.get(key)
if(priceMap){
if(priceMap.zhugezhaofang){
d['F'] = priceMap.zhugezhaofang
}
if(priceMap.anjuke){
d['G'] = priceMap.anjuke
}
if(priceMap.soufang){
d['H'] = priceMap.soufang
}
if(priceMap.lianjia){
d['I'] = priceMap.lianjia
}
if(priceMap.cityhouse){
d['J'] = priceMap.cityhouse
}
if(priceMap.fangtan){
d['K'] = priceMap.fangtan
}
if(priceMap.fangdoudou){
d['L'] = priceMap.fangdoudou
}
if(priceMap.maitian){
d['M'] = priceMap.maitian
}
if(priceMap.get("58tc")){
d['N'] = priceMap.get("58tc")
}
}
}, 0)
}
def run1(){
ExcelUtils.modify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供-结果.xlsx",{d->
if(d['B'] && !d['D']){
if(d['B'].contains("号")){
// def price = searchContent2("福州","https://fz.58.com", d['B'], 304)
// if(price){
// d['C'] = price
// d['F'] = "58同城"
// }
def price = thirdPartyColl("福州", d['B'])
if(price){
d['D'] = price;
d['F'] = "麦田"
}
}
}
},0)
}
def thirdPartyColl(def city, def address){
def q = new BasicDBObject()
q.put("city", city)
q.put("address", address)
q.put("source","麦田")
q.put("s_date",202202)
def obj = thirdPartyColl.findOne(q)
if(obj){
return obj.avg_price
}
return null;
}
//查看此关键字是否有数据
def searchContent2(def city,def url,def keyword, def cityid){
def newUrl = url+"/esf-ajax/community/pc/autocomplete/?city_id=${cityid}&kw="+URLEncoder.encode(keyword, "utf-8")+"&type=2"
def document = CrawalUtil.getDocumentByABuYun(newUrl, "访问过于频繁", "utf-8", null, true);
def price = null
if(document){
println "【"+keyword +"】 查询"
def jsonObj = JSONObject.fromObject(document)
if(jsonObj && jsonObj.status=="ok"){
if(jsonObj.data.size()>0){
def datas = jsonObj.data
def d = datas[0]
price = d.price.toInteger()
}
}else{
println "【"+keyword +"】 无结果"
}
}
return price;
}
def run(){
def map = findThirdPrice("福州")
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供.xlsx",{d->
if(!d['F']){
def lists = CommonDatautils.findDistrictsByCityAndName("福州",d['A'], "district")
if(!lists){
lists = CommonDatautils.findDistrictByAlias("福州",null, d['A'], "district")
}
for(def o : lists){
// if(o.address == d['B']){
def priceMap = map.get(o.region+"##"+o.name)
d['C'] = priceMap.anjuke_esf_price
d['D'] = priceMap.maitian_esf_price
d['E'] = priceMap.beike_esf_price
d['F'] = o.region
// }
}
}
},0)
}
def findThirdPrice(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', 202109))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor) {
def key = db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.get("lianjia_esf_price") && db.lianjia_esf_price>3000){
dataMap.put("beike_esf_price", db.lianjia_esf_price)
}
}
return map;
}
}
@@ -37,7 +37,7 @@ class YouDaDataParser {
public static final DBCollection youda_newhouse_coll = MongoUtils.getCollectionByName2("business", "res_newdealprice", offlineUrl)
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
//线下:228 线上:45
//线下:228 线上:45 测试
static def new_targetUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc"
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
@@ -48,7 +48,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-01-01"
static def statDate = "2022-02-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2022年2月\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -490,8 +490,9 @@ class YouDaDataParser {
if(s){
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
if(hits && hits.total instanceof Integer){
total = hits.total
}
}
return total
@@ -565,7 +566,7 @@ class YouDaDataParser {
if(obj.premisesName && obj.premisesName!="null"){
dbo.premisesName = obj.premisesName
}
dbo.price = obj.price
dbo.price = obj.price.toInteger()
dbo.region = obj.region
dbo.resId = obj.ydId.toString()
dbo.totalPrice = obj.totalPrice
@@ -36,7 +36,7 @@ public class BeiKeCrawlByChrome {
def task = new BeiKeCrawlByChrome();
task.run()
// task.run1()
// task.addDealRegion();
task.addDealRegion();
}
@@ -142,7 +142,7 @@ public class BeiKeCrawlByChrome {
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
}else{
@@ -342,9 +342,9 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("天津")){
if(e.text().contains("天津")){
cityMap.put(e.text(), "https:"+href)
// }
}
}
}
return cityMap
@@ -514,8 +514,9 @@ class GaoDeUtils {
// "2a1fb75990f2b39f6221e50892dd3988"
// ]
//35925efd7333f1df52145e51b8e4e56a 15021384995
def KeyList = [
"10bcb421d2ee4e51ed9268d6dca58ad1"
"10bcb421d2ee4e51ed9268d6dca58ad1"//17740818975
]
def count = RandomUtils.nextInt(KeyList.size())
return KeyList.get(count);
@@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-30", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-31", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -421,8 +421,7 @@ public class LianjiaCrawlByChrome {
def href = e.attr("href")
// if(!href.contains("fang.lianjia") && e.text().contains("福州")){
cityMap.put(e.text(), href)
// }
}
}
}
return cityMap
}
@@ -40,10 +40,10 @@ class DataStat {
def run(){
def source = "58同城"//
def citys = findCrawlCity(source, 202201);
def source = "搜房"//
def citys = findCrawlCity(source, 202112);
for(def city : citys){
def count1 = findCount(source, city, 202201);
def count1 = findCount(source, city, 202112);
def count2 = findCount(source, city, 202202);
println city +" "+count1+" "+count2
// println count1
File diff suppressed because one or more lines are too long
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
//"fang.com":"搜房",
def sourceMap = ["jiwu.com":"吉屋","ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
def sourceMap = ["fang.com":"搜房"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -26,7 +26,7 @@ class SynchDistrictPrice {
def run(){
def citys = CommonUtils.findAllCity()
def citys = ["乌鲁木齐"]//CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){