update
This commit is contained in:
+38
-38
@@ -52,47 +52,47 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def citys = ['南京', '无锡', '徐州', '常州', '苏州', '南通', '连云港', '淮安', '盐城', '扬州', '镇江', '泰州', '宿迁']//CcbCommonUtils.citys;
|
||||
// def ccbCitys = CcbCommonUtils.citys;
|
||||
|
||||
// for(def city : citys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
// }
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// for(def city : citys){
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成"
|
||||
|
||||
|
||||
}
|
||||
@@ -571,7 +571,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def addDealPriceMatch(city){
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20220101", "yyyyMMdd"))
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd"))
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
|
||||
|
||||
|
||||
+2
-2
@@ -373,10 +373,10 @@ class DealPriceDataProcess {
|
||||
def citys = CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202205
|
||||
def month = 202206
|
||||
def rows = []
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202206)
|
||||
def rets = findPeGteThirdParty(city, 202207)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
for(def m : rets){
|
||||
|
||||
+8
-2
@@ -24,7 +24,7 @@ class LoadGuide {
|
||||
|
||||
def run(){
|
||||
// addName("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx")
|
||||
addPersonGuided("D:\\数据\\建行\\房产估值\\指导价\\7月人工指导价.xlsx")
|
||||
addPersonGuided("D:\\数据\\建行\\房产估值\\指导价\\7月23日建行库第一批录入.xlsx")
|
||||
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
|
||||
@@ -61,6 +61,10 @@ class LoadGuide {
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
createSql( r, cityCodeDatas, distCodeMap)
|
||||
// def dist = CommonDatautils.findDistrictById(r['A'])
|
||||
// if(dist){
|
||||
// r['I'] = dist.region
|
||||
// }
|
||||
}
|
||||
|
||||
})
|
||||
@@ -79,6 +83,7 @@ class LoadGuide {
|
||||
//def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district")
|
||||
def dist = CommonDatautils.findDistrictById(commid)
|
||||
|
||||
def index = 0
|
||||
if(dist){
|
||||
|
||||
def sampleRegion = dist.region
|
||||
@@ -89,7 +94,8 @@ class LoadGuide {
|
||||
|
||||
def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " +
|
||||
"('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')"
|
||||
println sql
|
||||
index ++
|
||||
println index+" "+sql
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}else{
|
||||
println "区域未标准化"
|
||||
|
||||
+52
-25
@@ -45,9 +45,9 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
def url = "https://sh.anjuke.com"
|
||||
def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
|
||||
println s
|
||||
//def url = "https://sh.anjuke.com"
|
||||
//def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
|
||||
//println s
|
||||
|
||||
//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁")
|
||||
//for(def d : districts){
|
||||
@@ -326,28 +326,55 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d->
|
||||
// if(d['A']){
|
||||
//// def region = d['B']
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
//// if(newRegion) {
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("status", 1)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
|
||||
if(d['A']){
|
||||
// def region = d['B']
|
||||
// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
// if(newRegion) {
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dist._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d
|
||||
}
|
||||
// }
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
|
||||
if(d['A']){
|
||||
// def region = d['B']
|
||||
// def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
// if(newRegion) {
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dist._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d
|
||||
}
|
||||
// }
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
|
||||
|
||||
//def citys = CcbCommonUtils.citys
|
||||
//def rets = []
|
||||
|
||||
@@ -18,7 +18,7 @@ import com.mongodb.DBObject
|
||||
|
||||
|
||||
|
||||
println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk");
|
||||
//println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk");
|
||||
/**
|
||||
* Created by gaojuan on 2017/9/4.
|
||||
*/
|
||||
@@ -45,27 +45,53 @@ println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk"
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据")
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d->
|
||||
if(d['B']){
|
||||
def json = GaoDeApiUtils.geo("重庆", d['B'])
|
||||
if(json && json.info=="OK"){
|
||||
def datas = json.geocodes
|
||||
if(datas){
|
||||
def row = datas[0]
|
||||
println row
|
||||
d['C'] = row.country
|
||||
d['D'] = row.province
|
||||
d['E'] = row.city.toString().replace("[]","")
|
||||
d['F'] = row.citycode.toString().replace("[]","")
|
||||
d['G'] = row.district.toString().replace("[]","")
|
||||
d['H'] = row.street.toString().replace("[]","")
|
||||
d['I'] = row.number.toString().replace("[]","")
|
||||
d['J'] = row.adcode.toString().replace("[]","")
|
||||
d['K'] = row.location
|
||||
d['L'] = row.level
|
||||
}
|
||||
}
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d->
|
||||
// if(d['B']){
|
||||
// def json = GaoDeApiUtils.geo("重庆", d['B'])
|
||||
// if(json && json.info=="OK"){
|
||||
// def datas = json.geocodes
|
||||
// if(datas){
|
||||
// def row = datas[0]
|
||||
// println row
|
||||
// d['C'] = row.country
|
||||
// d['D'] = row.province
|
||||
// d['E'] = row.city.toString().replace("[]","")
|
||||
// d['F'] = row.citycode.toString().replace("[]","")
|
||||
// d['G'] = row.district.toString().replace("[]","")
|
||||
// d['H'] = row.street.toString().replace("[]","")
|
||||
// d['I'] = row.number.toString().replace("[]","")
|
||||
// d['J'] = row.adcode.toString().replace("[]","")
|
||||
// d['K'] = row.location
|
||||
// d['L'] = row.level
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
//})
|
||||
println "开始统计"
|
||||
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
def prices = map.get(d['B'])
|
||||
if(!prices){
|
||||
prices = []
|
||||
map.put(d['B'], prices)
|
||||
}
|
||||
prices << d['G']
|
||||
}
|
||||
|
||||
def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"]
|
||||
def rows = []
|
||||
rows << ["区域","数量", "均价"]
|
||||
|
||||
})
|
||||
regions.each{e->
|
||||
def prices = map.get(e)
|
||||
def price = (prices.sum()/prices.size()).toInteger()
|
||||
rows << [e, prices.size(), price]
|
||||
}
|
||||
//map.each{region,prices->
|
||||
// def price = (prices.sum()/prices.size()).toInteger()
|
||||
// rows << [region, prices.size(), price]
|
||||
//}
|
||||
ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果")
|
||||
+96
-10
@@ -1,38 +1,124 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class AnjukeDistSearch {
|
||||
static main(args) {
|
||||
def task = new AnjukeDistSearch()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def run1(){
|
||||
|
||||
def file = "D:\\数据\\朔州匹配房源1.xlsx"
|
||||
def i = 0
|
||||
|
||||
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
// System.setProperty("http.nonProxyHosts", "localhost");
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
|
||||
ExcelUtils.modify(file,{d->
|
||||
def data = getDataInfo(282, d['C'])
|
||||
if(data){
|
||||
def match = data.match
|
||||
if(match){
|
||||
println match
|
||||
i++
|
||||
def count = d['D'].toString().toDouble().toInteger()
|
||||
if(count <=3 && !d['E']){
|
||||
def url = " https://m.anjuke.com/shuo/community/?q="+URLEncoder.encode(d['C'], "utf-8")
|
||||
// def doc = getDocument(url, driver)
|
||||
def doc = ABuYunUtil.doGet(url, null, null, "utf-8")
|
||||
if(doc){
|
||||
def list = JQueryUtils.select(".c-community-list a", doc)
|
||||
for(def r : list){
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
doc = Jsoup.parse(s);
|
||||
Thread.sleep(5000)
|
||||
if(!doc || (s && s.contains("验证"))){
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def file = "D:\\数据\\朔州匹配房源1.xlsx"
|
||||
def i = 0
|
||||
ExcelUtils.modify(file,{d->
|
||||
i++
|
||||
def count = d['D'].toString().toDouble().toInteger()
|
||||
if(count <=3 && !d['E']){
|
||||
def data = getDataInfo(282, d['C'])
|
||||
if(data){
|
||||
def match = data.match
|
||||
def ids = []
|
||||
if(match){
|
||||
println i+" "+ match
|
||||
for(def r : match){
|
||||
ids << r.comm_id
|
||||
}
|
||||
}
|
||||
d['E'] = ids.join(",")
|
||||
}
|
||||
}
|
||||
})
|
||||
getDataInfo()
|
||||
}
|
||||
|
||||
def getDataInfo(cityId, keyword){
|
||||
def url = "https://m.anjuke.com/ajax/autocomplete/?city_id=282&kw="+URLEncoder.encode(keyword, "utf-8")
|
||||
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
|
||||
if (ret) {
|
||||
|
||||
def header = [:]
|
||||
header.put("accept","application/json, text/plain, */*")
|
||||
header.put("accept-encoding","gzip, deflate, br")
|
||||
header.put("accept-language","zh-CN,zh;q=0.9")
|
||||
// header.put("cookie","sessid=39D02194-D146-4BF9-A51E-79C24F4DE0FC; ajk-appVersion=; id58=CpQB3mLL+XJIv5mECb1VAg==; aQQ_ajkguid=02672143-A93A-42AF-B37F-C86984A83BEC; 58tj_uuid=0714884f-f17b-4145-b201-605539f483be; _ga=GA1.2.1000624676.1657534859; als=0; seo_source_type=0; twe=2; ctid=282; obtain_by=2; xxzl_cid=ef7523dbc335430f841f42cc67185e63; xzuid=56c772a8-8b4d-45c5-a356-e896e14efe43; fzq_h=68fb4a86e47dd3a3d8648011e8d6f05e_1658390102879_497b24518dd941868c10f254e7132af2_1917884102; new_session=1; init_refer=; new_uv=16")
|
||||
header.put("cookie","ew_uv=16")
|
||||
|
||||
header.put("referer","https://m.anjuke.com/shuo/")
|
||||
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1")
|
||||
|
||||
def ret = ABuYunUtil.doGet(url, null, header, "utf-8")
|
||||
if(ret && ret.contains("验证")){
|
||||
def num = new Random().nextInt()
|
||||
header.put("cookie","ew_uv="+num)
|
||||
ret = ABuYunUtil.doGet(url, null, header, "utf-8")
|
||||
}
|
||||
if (ret && !ret.contains("验证")) {
|
||||
JSONObject jsonObj = JSONObject.fromObject(ret);
|
||||
if (jsonObj && jsonObj.msg == "ok") {
|
||||
def data = jsonObj.data
|
||||
return data
|
||||
}
|
||||
}else{
|
||||
println keyword +" 验证页面"
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
+6
-2
@@ -25,11 +25,11 @@ class AnjukeHouseCrawl {
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
|
||||
def href = "https://zj.lianjia.com/ershoufang/103120144204.html"
|
||||
def href = "https://yz.esf.fang.com/loupan/1816376982/housedetail.htm"
|
||||
def doc = getDocument(href, driver)
|
||||
// def doc = ClaABuYun.getResponse(href)
|
||||
// def newUrl = href.substring(0, hrefs.indexOf("?"))
|
||||
sendMq(href, doc, "sell_lianjia")
|
||||
sendMq(href, doc, "community_fang")
|
||||
|
||||
// for(def i=25;i<50;i++){
|
||||
// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
|
||||
@@ -216,6 +216,10 @@ class AnjukeHouseCrawl {
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".main-wrap", doc)
|
||||
}
|
||||
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".con_item", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
|
||||
+302
-27
@@ -6,6 +6,9 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class ExportPaimai {
|
||||
|
||||
@@ -15,11 +18,27 @@ class ExportPaimai {
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportPaimai();
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.runCsv()
|
||||
// task.runCsv2()
|
||||
// task.addressStandard()
|
||||
// task.addPrice()
|
||||
task.stat()
|
||||
// task.stat()
|
||||
// task.standard()
|
||||
// task.updateCount()
|
||||
}
|
||||
|
||||
def updateCount(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\京东拍卖.xlsx",{d->
|
||||
if(d['AC']){
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("url",d['AC'])
|
||||
def obj = paimai43_COLL.findOne(pmquery)
|
||||
if(obj){
|
||||
d['R'] = obj.addPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def stat(){
|
||||
@@ -108,34 +127,77 @@ class ExportPaimai {
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def runCsv(){
|
||||
def datas = findPaimaiData()
|
||||
println "总量:"+datas.size()
|
||||
exportCsv("D:\\数据\\重庆农商行\\法拍\\淘宝法拍.csv", datas)
|
||||
}
|
||||
|
||||
def runCsv2(){
|
||||
def datas = findPaimaiData()
|
||||
println "总量:"+datas.size()
|
||||
exportCsv("D:\\数据\\重庆农商行\\法拍\\淘宝法拍.csv", datas)
|
||||
}
|
||||
|
||||
|
||||
def batchCreateFile(ret, fileName, headers, charset){
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
def sb = new StringBuffer();
|
||||
// sb.append(headers.join("@|@"))
|
||||
// sb.append(System.getProperty("line.separator"));
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "@|@"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
println index
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"开始写入CSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"结束写入CSV "+new Date().toLocaleString()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "宿迁"
|
||||
def region = null
|
||||
def datas = findPaimaiData(city,region)
|
||||
println "总量:"+datas.size()
|
||||
def rows = []
|
||||
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
|
||||
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
|
||||
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
|
||||
rows << header
|
||||
def set = new HashSet()
|
||||
// def set = new HashSet()
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
if(!set.contains(d.title)){
|
||||
// if(!set.contains(d.title)){
|
||||
index++
|
||||
println index
|
||||
set << d.title
|
||||
// set << d.title
|
||||
def row = []
|
||||
row << d.title
|
||||
row << d.title.replace(",",",")
|
||||
row << d.province
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.address
|
||||
row << ""
|
||||
// def formatAddress = AddressUtil.formatAddress(d.city, d.title)
|
||||
// if(formatAddress){
|
||||
// row << formatAddress
|
||||
// }else{
|
||||
// row << ""
|
||||
// }
|
||||
row << d.houseNum
|
||||
row << d.unitNum
|
||||
row << d.roomNum
|
||||
@@ -167,26 +229,239 @@ class ExportPaimai {
|
||||
row << d.url
|
||||
|
||||
rows << row
|
||||
}else{
|
||||
println d.title
|
||||
}
|
||||
// }else{
|
||||
// println d.title
|
||||
// }
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已流拍).xlsx",rows,"数据")
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}).xlsx",rows,"数据")
|
||||
|
||||
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
|
||||
println "生成完成........"
|
||||
}
|
||||
|
||||
def findPaimaiData(){
|
||||
|
||||
def exportCsv(path, list){
|
||||
File file = new File(path);
|
||||
if (file.exists()){
|
||||
file.delete();
|
||||
}
|
||||
file.append("标地物地址,")
|
||||
file.append("省,")
|
||||
file.append("城市,")
|
||||
file.append("区域,")
|
||||
file.append("地址,")
|
||||
file.append("标题提取地址,")
|
||||
file.append("楼栋号,")
|
||||
file.append("单元,")
|
||||
file.append("室号,")
|
||||
file.append("楼盘名称,")
|
||||
file.append("建筑面积,")
|
||||
file.append("竣工日期,")
|
||||
file.append("房屋用途,")
|
||||
file.append("总楼层,")
|
||||
file.append("拍卖状态,")
|
||||
file.append("是否成交,")
|
||||
file.append("拍卖阶段,")
|
||||
file.append("拍卖方式,")
|
||||
file.append("成交价,")
|
||||
file.append("拍卖开始时间,")
|
||||
file.append("拍卖结束时间,")
|
||||
file.append("起拍价,")
|
||||
file.append("保证金,")
|
||||
file.append("加价幅度,")
|
||||
file.append("评估价,")
|
||||
file.append("市场价,")
|
||||
file.append("变卖预缴款,")
|
||||
file.append("竞价周期,")
|
||||
file.append("延时周期,")
|
||||
file.append("优先购买权人,")
|
||||
file.append("报名人数,")
|
||||
file.append("围观人数,")
|
||||
file.append("设置提醒人数,")
|
||||
file.append("延迟次数,")
|
||||
file.append("URL,")
|
||||
file.append(System.getProperty("line.separator"));
|
||||
list.each{
|
||||
def d = it;
|
||||
file.append(d.title + ",","gbk")
|
||||
file.append(d.province + ",","gbk")
|
||||
file.append(d.city + ",","gbk")
|
||||
if(d.region){
|
||||
file.append(d.region+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
|
||||
}
|
||||
|
||||
if(d.address){
|
||||
file.append(d.address +",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
|
||||
}
|
||||
|
||||
file.append(",")
|
||||
|
||||
if(d.houseNum){
|
||||
file.append(d.houseNum +",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
|
||||
}
|
||||
if(d.unitNum){
|
||||
file.append(d.unitNum+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
|
||||
if(d.roomNum){
|
||||
file.append(d.roomNum+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
|
||||
if(d.matchName){
|
||||
file.append(d.matchName+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.area){
|
||||
file.append(d.area+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.buildYear){
|
||||
file.append(d.buildYear+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.estateType){
|
||||
file.append(d.estateType+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
file.append(",")
|
||||
file.append(",")
|
||||
|
||||
if(d.status){
|
||||
file.append(d.status+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
|
||||
if(d.stage){
|
||||
file.append(d.stage+",","gbk")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
file.append(",")
|
||||
|
||||
if(d.tradePrice){
|
||||
file.append(d.tradePrice+",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.startDate){
|
||||
file.append(DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd") +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.finishDate){
|
||||
file.append(DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd") +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.startPrice){
|
||||
file.append(d.startPrice +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.depositPrice){
|
||||
file.append(d.depositPrice +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
|
||||
if(d.addPrice){
|
||||
file.append(d.addPrice +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.evalatePrice){
|
||||
file.append(d.evalatePrice+",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.marketPrice){
|
||||
file.append(d.marketPrice +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.sellOffAdvancePay){
|
||||
file.append(d.sellOffAdvancePay +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.pricePkWeekly){
|
||||
file.append(d.pricePkWeekly +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.delayWeekly){
|
||||
file.append(d.delayWeekly +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.preemptivePerson){
|
||||
file.append(d.preemptivePerson +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.enrollCount){
|
||||
file.append(d.enrollCount +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.lookCount){
|
||||
file.append(d.lookCount +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.callCount){
|
||||
file.append(d.callCount +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.delayCount){
|
||||
file.append(d.delayCount +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
if(d.url){
|
||||
file.append(d.url +",")
|
||||
}else{
|
||||
file.append(",")
|
||||
}
|
||||
file.append(System.getProperty("line.separator"));
|
||||
}
|
||||
}
|
||||
|
||||
def findPaimaiData(city, region){
|
||||
def pmquery = new BasicDBObject()
|
||||
pmquery.put("city","苏州")
|
||||
pmquery.put("region","姑苏")
|
||||
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
|
||||
pmquery.put("status","已流拍")
|
||||
def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1))
|
||||
pmquery.put("city",city)
|
||||
if(region){
|
||||
pmquery.put("region",region)
|
||||
}
|
||||
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
|
||||
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
|
||||
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
|
||||
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
|
||||
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
pmquery.put("url", new BasicDBObject('$regex', pattern))
|
||||
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
// if(db.estateType && db.estateType=="车库"){
|
||||
// continue
|
||||
// }
|
||||
rets << db
|
||||
}
|
||||
return rets
|
||||
|
||||
+287
-33
@@ -11,11 +11,295 @@ import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
|
||||
class PaimaiDataSend {
|
||||
|
||||
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
|
||||
DBCollection ali_zc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_ali_zc_detail", "192.168.0.235", 27018)
|
||||
DBCollection ali_fp_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_ali_detail", "192.168.0.235", 27018)
|
||||
DBCollection jd_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_jd_detail", "192.168.0.235", 27018)
|
||||
DBCollection rmfysszc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_rmfysszc_detail", "192.168.0.235", 27018)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new PaimaiDataSend()
|
||||
task.jdRun()
|
||||
// task.aliZcRun()
|
||||
// task.aliZcRun2()
|
||||
// task.aliFpRun()
|
||||
// task.rmfysszcRun()
|
||||
// task.checkUrl()
|
||||
|
||||
}
|
||||
|
||||
|
||||
def checkUrl(){
|
||||
def q = new BasicDBObject()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("url", 1)
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(3000)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def pm = new BasicDBObject();
|
||||
pm.put("url", d.url)
|
||||
def obj = paimai43_COLL.findOne(pm)
|
||||
if(!obj){
|
||||
println d.url
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def rmfysszcRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml")
|
||||
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println "rm: "+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_rmfysszc")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
|
||||
// def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("sub_count", d.sub_count)
|
||||
result.put("data_json", d.json)
|
||||
result.put("province", d.province)
|
||||
result.put("city", d.city)
|
||||
result.put("bid_res_content", d.bid_res_content)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 2)
|
||||
udata.put("m_date", new Date())
|
||||
rmfysszc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
def jdRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","宿迁市")
|
||||
// q.put("url","https://paimai.jd.com/108245329")
|
||||
def dbcursor = jd_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println "jd:"+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_jd")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.desc_content)
|
||||
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("bid_content", bid_content.toString())
|
||||
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("content_json", d.content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content.content)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 3)
|
||||
udata.put("m_date", new Date())
|
||||
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
def aliZcRun(){
|
||||
def pm_q = new BasicDBObject()
|
||||
pm_q.put("city","苏州")
|
||||
pm_q.put("region","姑苏")
|
||||
// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm")
|
||||
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
// pm_q.put("url", new BasicDBObject('$regex', pattern))
|
||||
// pm_q.put("status", null)
|
||||
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
|
||||
println dbcursor.count()
|
||||
for(def d : dbcursor){
|
||||
def flag = sendAliZc(d.url)
|
||||
if(!flag){
|
||||
println "不存在:"+d.url
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def sendAliZc(def url){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", url)
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(3000).sort(new BasicDBObject("_id",-1))
|
||||
def flag = false
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
flag = true
|
||||
println "ali-zc: "+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao_zc")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("trades", bid_content)
|
||||
// result.put("condition", d.condition)
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("desc_content", d.desc_content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content)
|
||||
|
||||
result.put("province", d.province)
|
||||
result.put("city", d.city)
|
||||
result.put("region", d.region)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
return flag
|
||||
}
|
||||
|
||||
def aliFpRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","宿迁")
|
||||
q.put("url","https://sf-item.taobao.com/sf_item/631616791778.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
// q.put("parser_status", new BasicDBObject('$exists',false))
|
||||
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println "ali-fp: "+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("trades", bid_content)
|
||||
result.put("province", d.province)
|
||||
result.put("city", d.city)
|
||||
result.put("region", d.region)
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("desc_content", d.desc_content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content)
|
||||
json.put("result", result)
|
||||
|
||||
rets << json
|
||||
}
|
||||
|
||||
println "共:"+rets.size()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def json = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def aliZcRun2(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","宿迁")
|
||||
// q.put("region","姑苏")
|
||||
// q.put("url","https://zc-item.taobao.com/auction/670178084823.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println "ali-zc: "+index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao_zc")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("trades", bid_content)
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("desc_content", d.desc_content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content)
|
||||
|
||||
result.put("province", d.province)
|
||||
result.put("city", d.city)
|
||||
result.put("region", d.region)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 4)
|
||||
udata.put("m_date", new Date())
|
||||
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
|
||||
//Date date=new Date(1550455200000);
|
||||
@@ -36,37 +320,7 @@ DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "p
|
||||
// urls << o.url
|
||||
//}
|
||||
//println "待解析:"+urls
|
||||
def index = 0
|
||||
//for(def url : urls){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("url", url)
|
||||
q.put("update_time", new BasicDBObject('$gte',DateUtils.parseFromTightByPattern("2022-07-12", "yyyy-MM-dd")))
|
||||
def dbcursor = collection.find(q)
|
||||
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "paimai_taobao")
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
|
||||
result.put("trades", bid_content)
|
||||
result.put("condition", d.condition)
|
||||
result.put("sub_count", d.sub_count)
|
||||
def data_json = net.sf.json.JSONObject.fromObject(d.json)
|
||||
result.put("data_json", data_json)
|
||||
result.put("desc_content", d.desc_content)
|
||||
result.put("item_notice_content", d.item_notice_content)
|
||||
result.put("notice_detail_content", d.notice_detail_content)
|
||||
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
}
|
||||
//}
|
||||
|
||||
|
||||
|
||||
+100
-64
@@ -28,24 +28,56 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave
|
||||
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
|
||||
|
||||
//def q = new BasicDBObject()
|
||||
//q.put("city","重庆")
|
||||
//q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd")))
|
||||
//def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
//def map = [:]
|
||||
//for(def d : dbcursor){
|
||||
// def list = map.get(d.region)
|
||||
// if(!list){
|
||||
// list = []
|
||||
// map.put(d.region, list)
|
||||
|
||||
def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"]
|
||||
for(def city : citys){
|
||||
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\江苏数据统计.xlsx",{d->
|
||||
// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
|
||||
// if(keys){
|
||||
// d['E'] = keys.size()
|
||||
// }
|
||||
// def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd")
|
||||
// if(!list.contains(key)){
|
||||
// list << key
|
||||
// }
|
||||
//}
|
||||
|
||||
// def rets = findHousesByCity(d['A'], "sell", 20210701)
|
||||
if(d['A']){
|
||||
def count = findHouseCount(d['A'])
|
||||
d['F'] = count
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
|
||||
def findHouseCount(city){
|
||||
def count = 0
|
||||
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20210701 and city='${city}'", {
|
||||
count = it.count
|
||||
println("$city\t${it.count}")
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def getDealPrices(def city, coll){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd")))
|
||||
def dbcursor = coll.find(q)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
// def list = map.get(d.region)
|
||||
// if(!list){
|
||||
// list = []
|
||||
// map.put(d.region, list)
|
||||
// }
|
||||
def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd")
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
//def q = new BasicDBObject()
|
||||
//q.put("status","已成交")
|
||||
//q.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")))
|
||||
@@ -71,42 +103,42 @@ DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "regio
|
||||
//
|
||||
//})
|
||||
|
||||
def citys = ["锡林郭勒盟","保亭","塔城","陇南","海东","日喀则","迪庆州","忻州","昌吉","海西州","和田","克拉玛依","梅州","潮州","延安","楚雄","阿克苏","平凉","商洛","普洱","大兴安岭","白山","西双版纳州","临夏","榆林","阿勒泰","文昌","陵水","吐鲁番","庆阳","临沧","保山","黑河","喀什","白银","朔州","龙岩","韶关","揭阳","儋州","吕梁","来宾","渭南","肇庆","运城","丽江","德宏州","长治","黔南","张掖","金华","河源","邢台","晋中","宁德","曲靖","昭通","拉萨","玉溪","林芝","百色","仙桃","湘潭","滨州","随州","日照","呼伦贝尔","崇左","汕尾"]
|
||||
|
||||
def districts = CommonDatautils.findAllDistrictByCity("朔州","district",["city","region","name","alias"])
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println "小区量:"+districts.size()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(rMap.city, rMap.region, "sell", 20210701,rMap.name, rMap.mapping_alias)
|
||||
println totalCount.get() +" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
|
||||
// if(!ret){
|
||||
// ret = findThirdParty(rMap.city, rMap.region, rMap.name, AVERAGE_STATISTICS)
|
||||
//def citys = ["锡林郭勒盟","保亭","塔城","陇南","海东","日喀则","迪庆州","忻州","昌吉","海西州","和田","克拉玛依","梅州","潮州","延安","楚雄","阿克苏","平凉","商洛","普洱","大兴安岭","白山","西双版纳州","临夏","榆林","阿勒泰","文昌","陵水","吐鲁番","庆阳","临沧","保山","黑河","喀什","白银","朔州","龙岩","韶关","揭阳","儋州","吕梁","来宾","渭南","肇庆","运城","丽江","德宏州","长治","黔南","张掖","金华","河源","邢台","晋中","宁德","曲靖","昭通","拉萨","玉溪","林芝","百色","仙桃","湘潭","滨州","随州","日照","呼伦贝尔","崇左","汕尾"]
|
||||
//
|
||||
//def districts = CommonDatautils.findAllDistrictByCity("朔州","district",["city","region","name","alias"])
|
||||
//
|
||||
//def totalCount = new AtomicLong()
|
||||
//final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
//println "小区量:"+districts.size()
|
||||
//
|
||||
//def retMap = new ConcurrentHashMap()
|
||||
//for(def i=0;i<districts.size();i++){
|
||||
// final def rMap = districts[i];
|
||||
//
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try {
|
||||
// totalCount.addAndGet(1)
|
||||
// def ret = findHouses(rMap.city, rMap.region, "sell", 20210701,rMap.name, rMap.mapping_alias)
|
||||
// println totalCount.get() +" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
|
||||
//// if(!ret){
|
||||
//// ret = findThirdParty(rMap.city, rMap.region, rMap.name, AVERAGE_STATISTICS)
|
||||
//// }
|
||||
// if(ret && ret.size()<=3){
|
||||
// def key = rMap.city+"#"+rMap.region+"#"+rMap.name
|
||||
// retMap.put(key, ret.size())
|
||||
// }
|
||||
if(ret && ret.size()<=3){
|
||||
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
|
||||
retMap.put(key, ret.size())
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\房源小于4个以下的小区.xlsx", datas, "数量")
|
||||
//
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// };
|
||||
// })
|
||||
//}
|
||||
//latch.await()
|
||||
//ExcelUtils.write("D:\\数据\\房源小于4个以下的小区.xlsx", datas, "数量")
|
||||
|
||||
//def rets = findHousesByCity("朔州", "sell", 20210701)
|
||||
//def map = [:]
|
||||
@@ -157,13 +189,13 @@ ExcelUtils.write("D:\\
|
||||
//
|
||||
//})
|
||||
|
||||
def countMap = ["深圳":5,"中山":409,"贵阳":372,"南宁":1130,"台州":1325,"杭州":1177,"南昌":1369,"厦门":1370,"绍兴":1363,"徐州":1468,"长沙":1620,"东莞":1673,"郑州":1769,"遵义":1004,"温州":2012,"烟台":2080,"哈尔滨":5459,"湛江":392,"株洲":1068,"武汉":2486,"长春":2857,"岳阳":356,"肇庆":607,"镇江":1043,"芜湖":402,"银川":505,"合肥":2818,"宜昌":786,"淄博":2441,"佛山":3067,"马鞍山":227,"清远":537,"汕头":707,"洛阳":552,"济南":3456,"九江":138,"秦皇岛":1495,"宁波":3772,"乌鲁木齐":2273,"沧州":394,"泉州":4692,"廊坊":1120,"唐山":2626,"石家庄":4914,"漳州":4071,"张家口":357,"沈阳":5000,"青岛":5052,"柳州":2270,"安庆":756,"兰州":5399,"拉萨":784,"昆明":5436,"吉林":2274,"惠州":6206,"嘉兴":6142,"常德":963,"梅州":317,"绵阳":494,"海口":7883,"江门":663,"临沂":1473,"日照":2070,"襄阳":209,"湖州":3667,"平顶山":1501,"泰安":356,"大同":2814,"衡阳":241,"桂林":372,"临汾":372,"锦州":2875,"北海":2507,"赣州":1659,"商丘":644,"德州":639,"渭南":430,"咸阳":301,"聊城":932,"保定":3672,"许昌":266,"邢台":277,"潍坊":3743,"西宁":4002,"宝鸡":204,"邯郸":2361,"安阳":557,"济宁":2194,"龙岩":367,"太原":7524,"南充":568,"盐城":6512,"阜阳":198,"运城":462,"菏泽":320,"晋中":539,"新乡":768,"滁州":1025,"衡水":969,"信阳":1048,"长治":806,"韶关":696,"荆州":361,"汉中":118,"威海":3030,"孝感":984,"东营":874,"宜春":251,"开封":111,"驻马店":549,"湘潭":410,"黄冈":250,"三亚":7481,"上饶":164,"眉山":284,"绥化":851,"宁德":1469,"苏州":8876,"焦作":767,"牡丹江":1009,"齐齐哈尔":566,"枣庄":2636,"河源":167,"宿迁":3428,"遂宁":81,"滨州":3454,"漯河":308,"周口":1670,"茂名":347,"大庆":932,"平凉":210,"南阳":4087,"黄石":99,"鄂尔多斯":452,"营口":1470,"包头":4789,"吉安":442,"金华":13539,"通辽":238,"泸州":183,"乐山":2776,"佳木斯":506,"三门峡":574,"承德":2923,"荆门":64,"曲靖":1521,"忻州":770,"潮州":359,"永州":209,"揭阳":934,"呼伦贝尔":353,"扬州":12477,"宣城":293,"赤峰":2770,"吕梁":2618,"宜宾":322,"濮阳":2389,"宿州":264,"阳江":270,"延安":388,"丹东":885,"怀化":258,"黄山":312,"十堰":2687,"抚州":179,"郴州":4355,"衢州":1714,"陇南":4,"榆林":4694,"延边":1640,"抚顺":258,"丽水":817,"庆阳":2069,"玉溪":254,"玉林":544,"咸宁":1983,"舟山":1970,"达州":990,"亳州":289,"呼和浩特":10185,"张掖":64,"广元":148,"益阳":156,"汕尾":2392,"昌吉":225,"蚌埠":5496,"天水":173,"随州":211,"珠海":13972,"巴彦淖尔":0,"楚雄":95,"乌兰察布":172,"新余":86,"南平":7781,"广安":2129,"景德镇":474,"铁岭":2676,"松原":747,"淮安":13859,"三明":6425,"朝阳":3393,"酒泉":479,"四平":261,"梧州":248,"自贡":2983,"黔东南":205,"大理":1914,"黔南":866,"内江":177,"仙桃":70,"白银":2398,"朔州":1861,"娄底":2601,"六盘水":277,"百色":73,"阳泉":2135,"德阳":9330,"邵阳":2425,"盘锦":4269,"双鸭山":0,"白山":60,"通化":2379,"红河":4013,"鄂州":149,"吴忠":244,"毕节":88,"云浮":10,"伊犁":579,"丽江":331,"贵港":705,"黑河":10,"安康":381,"葫芦岛":6562,"资阳":1320,"鹤壁":1904,"萍乡":173,"锡林郭勒盟":34,"防城港":33,"钦州":2141,"阿克苏":540,"鹤岗":233,"铜川":12,"喀什":68,"巴中":2540,"凉山州":0,"鹰潭":46,"石嘴山":468,"保山":152,"伊春":54,"六安":7649,"定西":75,"鸡西":2384,"临夏":1598,"海西州":0,"商洛":58,"中卫":112,"普洱":302,"湘西":4,"来宾":8,"张家界":21,"西双版纳州":0,"恩施":2215,"攀枝花":87,"晋城":6613,"乌海":361,"七台河":172,"雅安":1152,"黔西南":391,"嘉峪关":37,"淮南":6767,"潜江":0,"辽源":163,"临沧":89,"塔城":1,"克拉玛依":121,"金昌":10,"儋州":33,"海东":106,"大兴安岭":5,"兴安盟":972,"和田":0,"吐鲁番":107,"迪庆州":14,"安顺":1939,"日喀则":32,"文昌":2,"林芝":9,"保亭":0,"陵水":0,"阿勒泰":177,"天门":370,"崇左":560,"莆田":12067,"白城":2422,"铜仁":2209,"河池":1926,"辽阳":4615,"昭通":1995,"哈密":2592,"本溪":4452,"武威":1885,"德宏州":1654,"大连":18125,"成都":17618,"博尔塔拉":2094,"贺州":2168,"池州":6831,"固原":4445,"阜新":6345,"鞍山":15277,"巴音郭楞":7481,"铜陵":9225,"泰州":26770,"南通":26075,"连云港":30835,"西安":28834,"淮北":16917,"北京":35728,"南京":32406,"无锡":48220,"广州":50209,"天津":63311,"上海":160627,"常州":155553,"重庆":221058]
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d->
|
||||
def count = countMap.get(d['A'])
|
||||
d['R'] = count
|
||||
})
|
||||
|
||||
println "完成"
|
||||
//def countMap = ["深圳":5,"中山":409,"贵阳":372,"南宁":1130,"台州":1325,"杭州":1177,"南昌":1369,"厦门":1370,"绍兴":1363,"徐州":1468,"长沙":1620,"东莞":1673,"郑州":1769,"遵义":1004,"温州":2012,"烟台":2080,"哈尔滨":5459,"湛江":392,"株洲":1068,"武汉":2486,"长春":2857,"岳阳":356,"肇庆":607,"镇江":1043,"芜湖":402,"银川":505,"合肥":2818,"宜昌":786,"淄博":2441,"佛山":3067,"马鞍山":227,"清远":537,"汕头":707,"洛阳":552,"济南":3456,"九江":138,"秦皇岛":1495,"宁波":3772,"乌鲁木齐":2273,"沧州":394,"泉州":4692,"廊坊":1120,"唐山":2626,"石家庄":4914,"漳州":4071,"张家口":357,"沈阳":5000,"青岛":5052,"柳州":2270,"安庆":756,"兰州":5399,"拉萨":784,"昆明":5436,"吉林":2274,"惠州":6206,"嘉兴":6142,"常德":963,"梅州":317,"绵阳":494,"海口":7883,"江门":663,"临沂":1473,"日照":2070,"襄阳":209,"湖州":3667,"平顶山":1501,"泰安":356,"大同":2814,"衡阳":241,"桂林":372,"临汾":372,"锦州":2875,"北海":2507,"赣州":1659,"商丘":644,"德州":639,"渭南":430,"咸阳":301,"聊城":932,"保定":3672,"许昌":266,"邢台":277,"潍坊":3743,"西宁":4002,"宝鸡":204,"邯郸":2361,"安阳":557,"济宁":2194,"龙岩":367,"太原":7524,"南充":568,"盐城":6512,"阜阳":198,"运城":462,"菏泽":320,"晋中":539,"新乡":768,"滁州":1025,"衡水":969,"信阳":1048,"长治":806,"韶关":696,"荆州":361,"汉中":118,"威海":3030,"孝感":984,"东营":874,"宜春":251,"开封":111,"驻马店":549,"湘潭":410,"黄冈":250,"三亚":7481,"上饶":164,"眉山":284,"绥化":851,"宁德":1469,"苏州":8876,"焦作":767,"牡丹江":1009,"齐齐哈尔":566,"枣庄":2636,"河源":167,"宿迁":3428,"遂宁":81,"滨州":3454,"漯河":308,"周口":1670,"茂名":347,"大庆":932,"平凉":210,"南阳":4087,"黄石":99,"鄂尔多斯":452,"营口":1470,"包头":4789,"吉安":442,"金华":13539,"通辽":238,"泸州":183,"乐山":2776,"佳木斯":506,"三门峡":574,"承德":2923,"荆门":64,"曲靖":1521,"忻州":770,"潮州":359,"永州":209,"揭阳":934,"呼伦贝尔":353,"扬州":12477,"宣城":293,"赤峰":2770,"吕梁":2618,"宜宾":322,"濮阳":2389,"宿州":264,"阳江":270,"延安":388,"丹东":885,"怀化":258,"黄山":312,"十堰":2687,"抚州":179,"郴州":4355,"衢州":1714,"陇南":4,"榆林":4694,"延边":1640,"抚顺":258,"丽水":817,"庆阳":2069,"玉溪":254,"玉林":544,"咸宁":1983,"舟山":1970,"达州":990,"亳州":289,"呼和浩特":10185,"张掖":64,"广元":148,"益阳":156,"汕尾":2392,"昌吉":225,"蚌埠":5496,"天水":173,"随州":211,"珠海":13972,"巴彦淖尔":0,"楚雄":95,"乌兰察布":172,"新余":86,"南平":7781,"广安":2129,"景德镇":474,"铁岭":2676,"松原":747,"淮安":13859,"三明":6425,"朝阳":3393,"酒泉":479,"四平":261,"梧州":248,"自贡":2983,"黔东南":205,"大理":1914,"黔南":866,"内江":177,"仙桃":70,"白银":2398,"朔州":1861,"娄底":2601,"六盘水":277,"百色":73,"阳泉":2135,"德阳":9330,"邵阳":2425,"盘锦":4269,"双鸭山":0,"白山":60,"通化":2379,"红河":4013,"鄂州":149,"吴忠":244,"毕节":88,"云浮":10,"伊犁":579,"丽江":331,"贵港":705,"黑河":10,"安康":381,"葫芦岛":6562,"资阳":1320,"鹤壁":1904,"萍乡":173,"锡林郭勒盟":34,"防城港":33,"钦州":2141,"阿克苏":540,"鹤岗":233,"铜川":12,"喀什":68,"巴中":2540,"凉山州":0,"鹰潭":46,"石嘴山":468,"保山":152,"伊春":54,"六安":7649,"定西":75,"鸡西":2384,"临夏":1598,"海西州":0,"商洛":58,"中卫":112,"普洱":302,"湘西":4,"来宾":8,"张家界":21,"西双版纳州":0,"恩施":2215,"攀枝花":87,"晋城":6613,"乌海":361,"七台河":172,"雅安":1152,"黔西南":391,"嘉峪关":37,"淮南":6767,"潜江":0,"辽源":163,"临沧":89,"塔城":1,"克拉玛依":121,"金昌":10,"儋州":33,"海东":106,"大兴安岭":5,"兴安盟":972,"和田":0,"吐鲁番":107,"迪庆州":14,"安顺":1939,"日喀则":32,"文昌":2,"林芝":9,"保亭":0,"陵水":0,"阿勒泰":177,"天门":370,"崇左":560,"莆田":12067,"白城":2422,"铜仁":2209,"河池":1926,"辽阳":4615,"昭通":1995,"哈密":2592,"本溪":4452,"武威":1885,"德宏州":1654,"大连":18125,"成都":17618,"博尔塔拉":2094,"贺州":2168,"池州":6831,"固原":4445,"阜新":6345,"鞍山":15277,"巴音郭楞":7481,"铜陵":9225,"泰州":26770,"南通":26075,"连云港":30835,"西安":28834,"淮北":16917,"北京":35728,"南京":32406,"无锡":48220,"广州":50209,"天津":63311,"上海":160627,"常州":155553,"重庆":221058]
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d->
|
||||
// def count = countMap.get(d['A'])
|
||||
// d['R'] = count
|
||||
//})
|
||||
//
|
||||
//println "完成"
|
||||
|
||||
|
||||
def findThirdParty(city, region, name, coll){
|
||||
@@ -255,21 +287,25 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
|
||||
// list << d
|
||||
//}
|
||||
//
|
||||
|
||||
//def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"]
|
||||
//def rows = []
|
||||
//rows << ["区域","小区量", "别名量", "地址别名量"]
|
||||
//map.each{region, list->
|
||||
//rows << ["城市","小区量", "别名量", "地址别名量"]
|
||||
//for(def c : citys){
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(c,"district",["city","region","name","alias", "addr_alias"])
|
||||
// def alias_count = 0
|
||||
// def addr_alias_count = 0
|
||||
// for(def r : list){
|
||||
// if(r.mapping_alias && r.mapping_alias.size()>0){
|
||||
// for(def r : districts){
|
||||
// if(r.alias && r.alias.size()>0){
|
||||
// alias_count ++
|
||||
// }
|
||||
// if(r.addr_alias && r.addr_alias.size()>0){
|
||||
// addr_alias_count++
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// rows << [region, list.size(), alias_count, addr_alias_count]
|
||||
// rows << [c, districts.size(), alias_count, addr_alias_count]
|
||||
//}
|
||||
//
|
||||
//ExcelUtils.write("D:\\数据\\重庆农商行\\重庆数据统计.xlsx", rows, "楼盘统计")
|
||||
//
|
||||
//
|
||||
//ExcelUtils.write("D:\\数据\\江苏\\江苏数据统计.xlsx", rows, "结果")
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
package com.fangjia.script.jiangshu
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
class JsBlankTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new JsBlankTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def path = "D:\\Êý¾Ý\\½ËÕ\\"
|
||||
}
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['Q'] && row['D']){
|
||||
def city = row['B']
|
||||
def address = row['D']
|
||||
def region = row['C']
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['R'] = jsonObject.get("id")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202205);
|
||||
def count2 = findTotalCount(source, 202206);
|
||||
def count1 = findTotalCount(source, 202206);
|
||||
def count2 = findTotalCount(source, 202207);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user