Merge remote-tracking branch 'origin/master'

This commit is contained in:
2022-08-08 16:52:26 +08:00
13 changed files with 977 additions and 227 deletions
@@ -670,6 +670,7 @@ public class WeiXinEvaluationController {
model.setUserTrades(userTrade);
if(fjtUser!=null && fjtUser.getRole()==17 && district.getCity().equals("上海")){
System.out.println("渤海用户:"+fjtUser.getEmail());
Map<String, Object> retMap = getLoanInfo(district, evaluation.getHouseArea());
if(retMap!=null && retMap.size()>0){
model.setLoanInfoMap(retMap);
@@ -1245,6 +1246,9 @@ public class WeiXinEvaluationController {
}
}
Integer yearAge = getBuildYear(district.getFinishBuildingDate());
if(pbObj!=null){
System.out.println("渤海银行:"+district.getName() +" 年限:"+yearAge+" Level:"+pbObj.getLevel()+" 所属板块:"+pbObj.getName());
}
Map<String, Object> retsMap = new HashMap<String, Object>();
if(pbObj!=null && pbObj.getLevel()!=null && pbObj.getLevel()==1 && yearAge!=null && yearAge<=25 && area>=60){
retsMap.put("type", "A");
@@ -1262,6 +1266,7 @@ public class WeiXinEvaluationController {
retsMap.put("type", "D");
retsMap.put("loan_ratio", 5);
}
return retsMap;
}
@@ -52,47 +52,47 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
def citys = ['南京', '无锡', '徐州', '常州', '苏州', '南通', '连云港', '淮安', '盐城', '扬州', '镇江', '泰州', '宿迁']//CcbCommonUtils.citys;
// def ccbCitys = CcbCommonUtils.citys;
// for(def city : citys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for(def city : citys){
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
//
// def trades = getCityData(city, statMap, ppeDists)
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成"
}
@@ -571,7 +571,7 @@ class BigdataDealBatch extends BaseBatch{
def addDealPriceMatch(city){
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20220101", "yyyyMMdd"))
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
@@ -373,10 +373,10 @@ class DealPriceDataProcess {
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202205
def month = 202206
def rows = []
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202206)
def rets = findPeGteThirdParty(city, 202207)
def total = 0
println city +" "+rets.size()
for(def m : rets){
@@ -24,7 +24,7 @@ class LoadGuide {
def run(){
// addName("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx")
addPersonGuided("D:\\数据\\建行\\房产估值\\指导价\\7月人工指导价.xlsx")
addPersonGuided("D:\\数据\\建行\\房产估值\\指导价\\7月23日建行库第一批录入.xlsx")
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
@@ -61,6 +61,10 @@ class LoadGuide {
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
createSql( r, cityCodeDatas, distCodeMap)
// def dist = CommonDatautils.findDistrictById(r['A'])
// if(dist){
// r['I'] = dist.region
// }
}
})
@@ -79,6 +83,7 @@ class LoadGuide {
//def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district")
def dist = CommonDatautils.findDistrictById(commid)
def index = 0
if(dist){
def sampleRegion = dist.region
@@ -89,7 +94,8 @@ class LoadGuide {
def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " +
"('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')"
println sql
index ++
println index+" "+sql
sqlInstance.executeUpdate(sql)
}else{
println "区域未标准化"
@@ -45,9 +45,9 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def url = "https://sh.anjuke.com"
def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
println s
//def url = "https://sh.anjuke.com"
//def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
//println s
//def districts = CommonDatautils.findAllFiledDistrict("district", "南宁")
//for(def d : districts){
@@ -326,28 +326,55 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
//
//},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d->
// if(d['A']){
//// def region = d['B']
//// def newRegion = RegionData.formatRegion(d['A'], region)
//// if(newRegion) {
//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("status", 1)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d
// }
//// }
// }
//},0)
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
if(d['A']){
// def region = d['B']
// def newRegion = RegionData.formatRegion(d['A'], region)
// if(newRegion) {
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dist._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d
}
// }
}
},0)
ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d->
if(d['A']){
// def region = d['B']
// def newRegion = RegionData.formatRegion(d['A'], region)
// if(newRegion) {
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dist._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d
}
// }
}
},0)
//def citys = CcbCommonUtils.citys
//def rets = []
@@ -18,7 +18,7 @@ import com.mongodb.DBObject
println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk");
//println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk");
/**
* Created by gaojuan on 2017/9/4.
*/
@@ -45,27 +45,53 @@ println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk"
//}
//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据")
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d->
if(d['B']){
def json = GaoDeApiUtils.geo("重庆", d['B'])
if(json && json.info=="OK"){
def datas = json.geocodes
if(datas){
def row = datas[0]
println row
d['C'] = row.country
d['D'] = row.province
d['E'] = row.city.toString().replace("[]","")
d['F'] = row.citycode.toString().replace("[]","")
d['G'] = row.district.toString().replace("[]","")
d['H'] = row.street.toString().replace("[]","")
d['I'] = row.number.toString().replace("[]","")
d['J'] = row.adcode.toString().replace("[]","")
d['K'] = row.location
d['L'] = row.level
}
}
//ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d->
// if(d['B']){
// def json = GaoDeApiUtils.geo("重庆", d['B'])
// if(json && json.info=="OK"){
// def datas = json.geocodes
// if(datas){
// def row = datas[0]
// println row
// d['C'] = row.country
// d['D'] = row.province
// d['E'] = row.city.toString().replace("[]","")
// d['F'] = row.citycode.toString().replace("[]","")
// d['G'] = row.district.toString().replace("[]","")
// d['H'] = row.street.toString().replace("[]","")
// d['I'] = row.number.toString().replace("[]","")
// d['J'] = row.adcode.toString().replace("[]","")
// d['K'] = row.location
// d['L'] = row.level
// }
// }
// }
//
//
//})
println "开始统计"
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\成交\\202202.xlsx",0,1)
def map = [:]
for(def d : datas){
def prices = map.get(d['B'])
if(!prices){
prices = []
map.put(d['B'], prices)
}
prices << d['G']
}
def regions = ["闵行","浦东","嘉定","普陀","青浦","静安","金山","松江","虹口","杨浦","奉贤","长宁","宝山","徐汇","崇明","黄浦"]
def rows = []
rows << ["区域","数量", "均价"]
})
regions.each{e->
def prices = map.get(e)
def price = (prices.sum()/prices.size()).toInteger()
rows << [e, prices.size(), price]
}
//map.each{region,prices->
// def price = (prices.sum()/prices.size()).toInteger()
// rows << [region, prices.size(), price]
//}
ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\成交\\trend-2202.xlsx", rows, "结果")
@@ -1,38 +1,124 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeDistSearch {
static main(args) {
def task = new AnjukeDistSearch()
task.run()
// task.run()
task.run1()
}
def run(){
def run1(){
def file = "D:\\数据\\朔州匹配房源1.xlsx"
def i = 0
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
// System.setProperty("http.nonProxyHosts", "localhost");
// WebDriver driver = new ChromeDriver()
ExcelUtils.modify(file,{d->
def data = getDataInfo(282, d['C'])
if(data){
def match = data.match
if(match){
println match
i++
def count = d['D'].toString().toDouble().toInteger()
if(count <=3 && !d['E']){
def url = " https://m.anjuke.com/shuo/community/?q="+URLEncoder.encode(d['C'], "utf-8")
// def doc = getDocument(url, driver)
def doc = ABuYunUtil.doGet(url, null, null, "utf-8")
if(doc){
def list = JQueryUtils.select(".c-community-list a", doc)
for(def r : list){
}
}
}
})
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
def s = driver.getPageSource()
doc = Jsoup.parse(s);
Thread.sleep(5000)
if(!doc || (s && s.contains("验证"))){
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def run(){
def file = "D:\\数据\\朔州匹配房源1.xlsx"
def i = 0
ExcelUtils.modify(file,{d->
i++
def count = d['D'].toString().toDouble().toInteger()
if(count <=3 && !d['E']){
def data = getDataInfo(282, d['C'])
if(data){
def match = data.match
def ids = []
if(match){
println i+" "+ match
for(def r : match){
ids << r.comm_id
}
}
d['E'] = ids.join(",")
}
}
})
getDataInfo()
}
def getDataInfo(cityId, keyword){
def url = "https://m.anjuke.com/ajax/autocomplete/?city_id=282&kw="+URLEncoder.encode(keyword, "utf-8")
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
if (ret) {
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
// header.put("cookie","sessid=39D02194-D146-4BF9-A51E-79C24F4DE0FC; ajk-appVersion=; id58=CpQB3mLL+XJIv5mECb1VAg==; aQQ_ajkguid=02672143-A93A-42AF-B37F-C86984A83BEC; 58tj_uuid=0714884f-f17b-4145-b201-605539f483be; _ga=GA1.2.1000624676.1657534859; als=0; seo_source_type=0; twe=2; ctid=282; obtain_by=2; xxzl_cid=ef7523dbc335430f841f42cc67185e63; xzuid=56c772a8-8b4d-45c5-a356-e896e14efe43; fzq_h=68fb4a86e47dd3a3d8648011e8d6f05e_1658390102879_497b24518dd941868c10f254e7132af2_1917884102; new_session=1; init_refer=; new_uv=16")
header.put("cookie","ew_uv=16")
header.put("referer","https://m.anjuke.com/shuo/")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1")
def ret = ABuYunUtil.doGet(url, null, header, "utf-8")
if(ret && ret.contains("验证")){
def num = new Random().nextInt()
header.put("cookie","ew_uv="+num)
ret = ABuYunUtil.doGet(url, null, header, "utf-8")
}
if (ret && !ret.contains("验证")) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if (jsonObj && jsonObj.msg == "ok") {
def data = jsonObj.data
return data
}
}else{
println keyword +" 验证页面"
}
return null
}
@@ -25,11 +25,11 @@ class AnjukeHouseCrawl {
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def href = "https://zj.lianjia.com/ershoufang/103120144204.html"
def href = "https://yz.esf.fang.com/loupan/1816376982/housedetail.htm"
def doc = getDocument(href, driver)
// def doc = ClaABuYun.getResponse(href)
// def newUrl = href.substring(0, hrefs.indexOf("?"))
sendMq(href, doc, "sell_lianjia")
sendMq(href, doc, "community_fang")
// for(def i=25;i<50;i++){
// def url = "https://fz.anjuke.com/sale/jinana/p${i}/"
@@ -216,6 +216,10 @@ class AnjukeHouseCrawl {
if(!els){
els = JQueryUtils.select(".main-wrap", doc)
}
if(!els){
els = JQueryUtils.select(".con_item", doc)
}
if(els && els.size()>0){
stop = true;
}else{
@@ -6,6 +6,9 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
class ExportPaimai {
@@ -15,11 +18,27 @@ class ExportPaimai {
static main(args) {
def task = new ExportPaimai();
// task.run()
task.run()
// task.runCsv()
// task.runCsv2()
// task.addressStandard()
// task.addPrice()
task.stat()
// task.stat()
// task.standard()
// task.updateCount()
}
def updateCount(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\法拍\\京东拍卖.xlsx",{d->
if(d['AC']){
def pmquery = new BasicDBObject()
pmquery.put("url",d['AC'])
def obj = paimai43_COLL.findOne(pmquery)
if(obj){
d['R'] = obj.addPrice
}
}
})
}
def stat(){
@@ -108,34 +127,77 @@ class ExportPaimai {
})
}
def run(){
def runCsv(){
def datas = findPaimaiData()
println "总量:"+datas.size()
exportCsv("D:\\数据\\重庆农商行\\法拍\\淘宝法拍.csv", datas)
}
def runCsv2(){
def datas = findPaimaiData()
println "总量:"+datas.size()
exportCsv("D:\\数据\\重庆农商行\\法拍\\淘宝法拍.csv", datas)
}
def batchCreateFile(ret, fileName, headers, charset){
OutputStream out = new FileOutputStream(fileName);
def sb = new StringBuffer();
// sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator"));
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row.toArray(), "@|@"))
sb.append(System.getProperty("line.separator"));
println index
if(index%10000==0){
println index+"开始写入CSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"结束写入CSV "+new Date().toLocaleString()
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
def run(){
def city = "宿迁"
def region = null
def datas = findPaimaiData(city,region)
println "总量:"+datas.size()
def rows = []
def header = ["标地物地址","省","城市","区域","地址","标题提取地址","楼栋号","单元","室号","楼盘名称","建筑面积","竣工日期","房屋用途","总楼层",
"拍卖状态","是否成交","拍卖阶段","拍卖方式","成交价","拍卖开始时间","拍卖结束时间","起拍价","保证金","加价幅度","评估价","市场价",
"变卖预缴款","竞价周期","延时周期","优先购买权人","报名人数","围观人数","设置提醒人数","延迟次数","URL"]
rows << header
def set = new HashSet()
// def set = new HashSet()
def index = 0
for(def d : datas){
if(!set.contains(d.title)){
// if(!set.contains(d.title)){
index++
println index
set << d.title
// set << d.title
def row = []
row << d.title
row << d.title.replace(",",",")
row << d.province
row << d.city
row << d.region
row << d.address
row << ""
// def formatAddress = AddressUtil.formatAddress(d.city, d.title)
// if(formatAddress){
// row << formatAddress
// }else{
// row << ""
// }
row << d.houseNum
row << d.unitNum
row << d.roomNum
@@ -167,26 +229,239 @@ class ExportPaimai {
row << d.url
rows << row
}else{
println d.title
}
// }else{
// println d.title
// }
}
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\苏州商业法拍原数据(已流拍).xlsx",rows,"数据")
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\京东拍卖(${city}).xlsx",rows,"数据")
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
println "生成完成........"
}
def findPaimaiData(){
def exportCsv(path, list){
File file = new File(path);
if (file.exists()){
file.delete();
}
file.append("标地物地址,")
file.append("省,")
file.append("城市,")
file.append("区域,")
file.append("地址,")
file.append("标题提取地址,")
file.append("楼栋号,")
file.append("单元,")
file.append("室号,")
file.append("楼盘名称,")
file.append("建筑面积,")
file.append("竣工日期,")
file.append("房屋用途,")
file.append("总楼层,")
file.append("拍卖状态,")
file.append("是否成交,")
file.append("拍卖阶段,")
file.append("拍卖方式,")
file.append("成交价,")
file.append("拍卖开始时间,")
file.append("拍卖结束时间,")
file.append("起拍价,")
file.append("保证金,")
file.append("加价幅度,")
file.append("评估价,")
file.append("市场价,")
file.append("变卖预缴款,")
file.append("竞价周期,")
file.append("延时周期,")
file.append("优先购买权人,")
file.append("报名人数,")
file.append("围观人数,")
file.append("设置提醒人数,")
file.append("延迟次数,")
file.append("URL,")
file.append(System.getProperty("line.separator"));
list.each{
def d = it;
file.append(d.title + ",","gbk")
file.append(d.province + ",","gbk")
file.append(d.city + ",","gbk")
if(d.region){
file.append(d.region+",","gbk")
}else{
file.append(",")
}
if(d.address){
file.append(d.address +",","gbk")
}else{
file.append(",")
}
file.append(",")
if(d.houseNum){
file.append(d.houseNum +",","gbk")
}else{
file.append(",")
}
if(d.unitNum){
file.append(d.unitNum+",","gbk")
}else{
file.append(",")
}
if(d.roomNum){
file.append(d.roomNum+",","gbk")
}else{
file.append(",")
}
if(d.matchName){
file.append(d.matchName+",","gbk")
}else{
file.append(",")
}
if(d.area){
file.append(d.area+",","gbk")
}else{
file.append(",")
}
if(d.buildYear){
file.append(d.buildYear+",","gbk")
}else{
file.append(",")
}
if(d.estateType){
file.append(d.estateType+",","gbk")
}else{
file.append(",")
}
file.append(",")
file.append(",")
if(d.status){
file.append(d.status+",","gbk")
}else{
file.append(",")
}
if(d.stage){
file.append(d.stage+",","gbk")
}else{
file.append(",")
}
file.append(",")
if(d.tradePrice){
file.append(d.tradePrice+",")
}else{
file.append(",")
}
if(d.startDate){
file.append(DateUtils.formatDateByPattern(d.startDate, "yyyy-MM-dd") +",")
}else{
file.append(",")
}
if(d.finishDate){
file.append(DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd") +",")
}else{
file.append(",")
}
if(d.startPrice){
file.append(d.startPrice +",")
}else{
file.append(",")
}
if(d.depositPrice){
file.append(d.depositPrice +",")
}else{
file.append(",")
}
if(d.addPrice){
file.append(d.addPrice +",")
}else{
file.append(",")
}
if(d.evalatePrice){
file.append(d.evalatePrice+",")
}else{
file.append(",")
}
if(d.marketPrice){
file.append(d.marketPrice +",")
}else{
file.append(",")
}
if(d.sellOffAdvancePay){
file.append(d.sellOffAdvancePay +",")
}else{
file.append(",")
}
if(d.pricePkWeekly){
file.append(d.pricePkWeekly +",")
}else{
file.append(",")
}
if(d.delayWeekly){
file.append(d.delayWeekly +",")
}else{
file.append(",")
}
if(d.preemptivePerson){
file.append(d.preemptivePerson +",")
}else{
file.append(",")
}
if(d.enrollCount){
file.append(d.enrollCount +",")
}else{
file.append(",")
}
if(d.lookCount){
file.append(d.lookCount +",")
}else{
file.append(",")
}
if(d.callCount){
file.append(d.callCount +",")
}else{
file.append(",")
}
if(d.delayCount){
file.append(d.delayCount +",")
}else{
file.append(",")
}
if(d.url){
file.append(d.url +",")
}else{
file.append(",")
}
file.append(System.getProperty("line.separator"));
}
}
def findPaimaiData(city, region){
def pmquery = new BasicDBObject()
pmquery.put("city","苏州")
pmquery.put("region","姑苏")
pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-07-01")))
pmquery.put("status","已流拍")
def dbcursor = paimai43_COLL.find(pmquery).sort(new BasicDBObject("finishDate",-1))
pmquery.put("city",city)
if(region){
pmquery.put("region",region)
}
// pmquery.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2019-01-01")))
pmquery.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// Pattern pattern = Pattern.compile("rmfysszc.gov.cn", Pattern.MULTILINE);
Pattern pattern = Pattern.compile("jd.com", Pattern.MULTILINE);
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
pmquery.put("url", new BasicDBObject('$regex', pattern))
def dbcursor = paimai43_COLL.find(pmquery).batchSize(2000)
def rets = []
for(def db : dbcursor){
// if(db.estateType && db.estateType=="车库"){
// continue
// }
rets << db
}
return rets
@@ -11,11 +11,295 @@ import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
class PaimaiDataSend {
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "ali_fapai_detail", "192.168.0.235", 27018)
DBCollection ali_zc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_ali_zc_detail", "192.168.0.235", 27018)
DBCollection ali_fp_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_ali_detail", "192.168.0.235", 27018)
DBCollection jd_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_jd_detail", "192.168.0.235", 27018)
DBCollection rmfysszc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_rmfysszc_detail", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new PaimaiDataSend()
task.jdRun()
// task.aliZcRun()
// task.aliZcRun2()
// task.aliFpRun()
// task.rmfysszcRun()
// task.checkUrl()
}
def checkUrl(){
def q = new BasicDBObject()
def keys = new BasicDBObject()
keys.put("url", 1)
def dbcursor = ali_fp_coll.find(q).batchSize(3000)
def index = 0
for(def d : dbcursor){
index++
def pm = new BasicDBObject();
pm.put("url", d.url)
def obj = paimai43_COLL.findOne(pm)
if(!obj){
println d.url
}
}
}
def rmfysszcRun(){
def index = 0
def q = new BasicDBObject()
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/76602.shtml")
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
println "rm: "+index
JSON json = new JSONObject()
json.put("project", "paimai_rmfysszc")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
// def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("sub_count", d.sub_count)
result.put("data_json", d.json)
result.put("province", d.province)
result.put("city", d.city)
result.put("bid_res_content", d.bid_res_content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 2)
udata.put("m_date", new Date())
rmfysszc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
}
def jdRun(){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁市")
// q.put("url","https://paimai.jd.com/108245329")
def dbcursor = jd_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
println "jd:"+index
JSON json = new JSONObject()
json.put("project", "paimai_jd")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.desc_content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("bid_content", bid_content.toString())
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("content_json", d.content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content.content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 3)
udata.put("m_date", new Date())
jd_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
}
def aliZcRun(){
def pm_q = new BasicDBObject()
pm_q.put("city","苏州")
pm_q.put("region","姑苏")
// pm_q.put("url","https://sf-item.taobao.com/sf_item/596573117068.htm")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("status", null)
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
def flag = sendAliZc(d.url)
if(!flag){
println "不存在:"+d.url
}
}
}
def sendAliZc(def url){
def index = 0
def q = new BasicDBObject()
q.put("url", url)
def dbcursor = ali_zc_coll.find(q).batchSize(3000).sort(new BasicDBObject("_id",-1))
def flag = false
for(def d : dbcursor){
index++
flag = true
println "ali-zc: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao_zc")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
// result.put("condition", d.condition)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
result.put("province", d.province)
result.put("city", d.city)
result.put("region", d.region)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
return flag
}
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁")
q.put("url","https://sf-item.taobao.com/sf_item/631616791778.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
def dbcursor = ali_fp_coll.find(q).batchSize(2000)
def rets = []
for(def d : dbcursor){
index++
println "ali-fp: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("province", d.province)
result.put("city", d.city)
result.put("region", d.region)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
rets << json
}
println "共:"+rets.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def json = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def aliZcRun2(){
def index = 0
def q = new BasicDBObject()
q.put("city","宿迁")
// q.put("region","姑苏")
// q.put("url","https://zc-item.taobao.com/auction/670178084823.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000)
for(def d : dbcursor){
index++
println "ali-zc: "+index
JSON json = new JSONObject()
json.put("project", "paimai_taobao_zc")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
result.put("province", d.province)
result.put("city", d.city)
result.put("region", d.region)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 4)
udata.put("m_date", new Date())
ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata))
}
}
}
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
//Date date=new Date(1550455200000);
@@ -36,37 +320,7 @@ DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "p
// urls << o.url
//}
//println "待解析:"+urls
def index = 0
//for(def url : urls){
def q = new BasicDBObject()
// q.put("url", url)
q.put("update_time", new BasicDBObject('$gte',DateUtils.parseFromTightByPattern("2022-07-12", "yyyy-MM-dd")))
def dbcursor = collection.find(q)
for(def d : dbcursor){
index++
println index
JSON json = new JSONObject()
json.put("project", "paimai_taobao")
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content)
result.put("trades", bid_content)
result.put("condition", d.condition)
result.put("sub_count", d.sub_count)
def data_json = net.sf.json.JSONObject.fromObject(d.json)
result.put("data_json", data_json)
result.put("desc_content", d.desc_content)
result.put("item_notice_content", d.item_notice_content)
result.put("notice_detail_content", d.notice_detail_content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
}
//}
@@ -28,24 +28,56 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
//def q = new BasicDBObject()
//q.put("city","重庆")
//q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd")))
//def dbcursor = DEAL_PRICE_COLL.find(q)
//def map = [:]
//for(def d : dbcursor){
// def list = map.get(d.region)
// if(!list){
// list = []
// map.put(d.region, list)
def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"]
for(def city : citys){
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\江苏数据统计.xlsx",{d->
// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
// if(keys){
// d['E'] = keys.size()
// }
// def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd")
// if(!list.contains(key)){
// list << key
// }
//}
// def rets = findHousesByCity(d['A'], "sell", 20210701)
if(d['A']){
def count = findHouseCount(d['A'])
d['F'] = count
}
})
def findHouseCount(city){
def count = 0
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20210701 and city='${city}'", {
count = it.count
println("$city\t${it.count}")
})
return count
}
def getDealPrices(def city, coll){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd")))
def dbcursor = coll.find(q)
def keys = []
for(def d : dbcursor){
// def list = map.get(d.region)
// if(!list){
// list = []
// map.put(d.region, list)
// }
def key = d.district_name+"##"+d.area+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date, "yyyyMMdd")
if(!keys.contains(key)){
keys << key
}
}
return keys;
}
//def q = new BasicDBObject()
//q.put("status","已成交")
//q.put("startDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")))
@@ -71,42 +103,42 @@ DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "regio
//
//})
def citys = ["锡林郭勒盟","保亭","塔城","陇南","海东","日喀则","迪庆州","忻州","昌吉","海西州","和田","克拉玛依","梅州","潮州","延安","楚雄","阿克苏","平凉","商洛","普洱","大兴安岭","白山","西双版纳州","临夏","榆林","阿勒泰","文昌","陵水","吐鲁番","庆阳","临沧","保山","黑河","喀什","白银","朔州","龙岩","韶关","揭阳","儋州","吕梁","来宾","渭南","肇庆","运城","丽江","德宏州","长治","黔南","张掖","金华","河源","邢台","晋中","宁德","曲靖","昭通","拉萨","玉溪","林芝","百色","仙桃","湘潭","滨州","随州","日照","呼伦贝尔","崇左","汕尾"]
def districts = CommonDatautils.findAllDistrictByCity("朔州","district",["city","region","name","alias"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println "小区量:"+districts.size()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(rMap.city, rMap.region, "sell", 20210701,rMap.name, rMap.mapping_alias)
println totalCount.get() +" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
// if(!ret){
// ret = findThirdParty(rMap.city, rMap.region, rMap.name, AVERAGE_STATISTICS)
//def citys = ["锡林郭勒盟","保亭","塔城","陇南","海东","日喀则","迪庆州","忻州","昌吉","海西州","和田","克拉玛依","梅州","潮州","延安","楚雄","阿克苏","平凉","商洛","普洱","大兴安岭","白山","西双版纳州","临夏","榆林","阿勒泰","文昌","陵水","吐鲁番","庆阳","临沧","保山","黑河","喀什","白银","朔州","龙岩","韶关","揭阳","儋州","吕梁","来宾","渭南","肇庆","运城","丽江","德宏州","长治","黔南","张掖","金华","河源","邢台","晋中","宁德","曲靖","昭通","拉萨","玉溪","林芝","百色","仙桃","湘潭","滨州","随州","日照","呼伦贝尔","崇左","汕尾"]
//
//def districts = CommonDatautils.findAllDistrictByCity("朔州","district",["city","region","name","alias"])
//
//def totalCount = new AtomicLong()
//final CountDownLatch latch = new CountDownLatch(districts.size())
//println "小区量:"+districts.size()
//
//def retMap = new ConcurrentHashMap()
//for(def i=0;i<districts.size();i++){
// final def rMap = districts[i];
//
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
// totalCount.addAndGet(1)
// def ret = findHouses(rMap.city, rMap.region, "sell", 20210701,rMap.name, rMap.mapping_alias)
// println totalCount.get() +" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
//// if(!ret){
//// ret = findThirdParty(rMap.city, rMap.region, rMap.name, AVERAGE_STATISTICS)
//// }
// if(ret && ret.size()<=3){
// def key = rMap.city+"#"+rMap.region+"#"+rMap.name
// retMap.put(key, ret.size())
// }
if(ret && ret.size()<=3){
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
retMap.put(key, ret.size())
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
ExcelUtils.write("D:\\数据\\房源小于4个以下的小区.xlsx", datas, "数量")
//
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// };
// })
//}
//latch.await()
//ExcelUtils.write("D:\\数据\\房源小于4个以下的小区.xlsx", datas, "数量")
//def rets = findHousesByCity("朔州", "sell", 20210701)
//def map = [:]
@@ -157,13 +189,13 @@ ExcelUtils.write("D:\\
//
//})
def countMap = ["深圳":5,"中山":409,"贵阳":372,"南宁":1130,"台州":1325,"杭州":1177,"南昌":1369,"厦门":1370,"绍兴":1363,"徐州":1468,"长沙":1620,"东莞":1673,"郑州":1769,"遵义":1004,"温州":2012,"烟台":2080,"哈尔滨":5459,"湛江":392,"株洲":1068,"武汉":2486,"长春":2857,"岳阳":356,"肇庆":607,"镇江":1043,"芜湖":402,"银川":505,"合肥":2818,"宜昌":786,"淄博":2441,"佛山":3067,"马鞍山":227,"清远":537,"汕头":707,"洛阳":552,"济南":3456,"九江":138,"秦皇岛":1495,"宁波":3772,"乌鲁木齐":2273,"沧州":394,"泉州":4692,"廊坊":1120,"唐山":2626,"石家庄":4914,"漳州":4071,"张家口":357,"沈阳":5000,"青岛":5052,"柳州":2270,"安庆":756,"兰州":5399,"拉萨":784,"昆明":5436,"吉林":2274,"惠州":6206,"嘉兴":6142,"常德":963,"梅州":317,"绵阳":494,"海口":7883,"江门":663,"临沂":1473,"日照":2070,"襄阳":209,"湖州":3667,"平顶山":1501,"泰安":356,"大同":2814,"衡阳":241,"桂林":372,"临汾":372,"锦州":2875,"北海":2507,"赣州":1659,"商丘":644,"德州":639,"渭南":430,"咸阳":301,"聊城":932,"保定":3672,"许昌":266,"邢台":277,"潍坊":3743,"西宁":4002,"宝鸡":204,"邯郸":2361,"安阳":557,"济宁":2194,"龙岩":367,"太原":7524,"南充":568,"盐城":6512,"阜阳":198,"运城":462,"菏泽":320,"晋中":539,"新乡":768,"滁州":1025,"衡水":969,"信阳":1048,"长治":806,"韶关":696,"荆州":361,"汉中":118,"威海":3030,"孝感":984,"东营":874,"宜春":251,"开封":111,"驻马店":549,"湘潭":410,"黄冈":250,"三亚":7481,"上饶":164,"眉山":284,"绥化":851,"宁德":1469,"苏州":8876,"焦作":767,"牡丹江":1009,"齐齐哈尔":566,"枣庄":2636,"河源":167,"宿迁":3428,"遂宁":81,"滨州":3454,"漯河":308,"周口":1670,"茂名":347,"大庆":932,"平凉":210,"南阳":4087,"黄石":99,"鄂尔多斯":452,"营口":1470,"包头":4789,"吉安":442,"金华":13539,"通辽":238,"泸州":183,"乐山":2776,"佳木斯":506,"三门峡":574,"承德":2923,"荆门":64,"曲靖":1521,"忻州":770,"潮州":359,"永州":209,"揭阳":934,"呼伦贝尔":353,"扬州":12477,"宣城":293,"赤峰":2770,"吕梁":2618,"宜宾":322,"濮阳":2389,"宿州":264,"阳江":270,"延安":388,"丹东":885,"怀化":258,"黄山":312,"十堰":2687,"抚州":179,"郴州":4355,"衢州":1714,"陇南":4,"榆林":4694,"延边":1640,"抚顺":258,"丽水":817,"庆阳":2069,"玉溪":254,"玉林":544,"咸宁":1983,"舟山":1970,"达州":990,"亳州":289,"呼和浩特":10185,"张掖":64,"广元":148,"益阳":156,"汕尾":2392,"昌吉":225,"蚌埠":5496,"天水":173,"随州":211,"珠海":13972,"巴彦淖尔":0,"楚雄":95,"乌兰察布":172,"新余":86,"南平":7781,"广安":2129,"景德镇":474,"铁岭":2676,"松原":747,"淮安":13859,"三明":6425,"朝阳":3393,"酒泉":479,"四平":261,"梧州":248,"自贡":2983,"黔东南":205,"大理":1914,"黔南":866,"内江":177,"仙桃":70,"白银":2398,"朔州":1861,"娄底":2601,"六盘水":277,"百色":73,"阳泉":2135,"德阳":9330,"邵阳":2425,"盘锦":4269,"双鸭山":0,"白山":60,"通化":2379,"红河":4013,"鄂州":149,"吴忠":244,"毕节":88,"云浮":10,"伊犁":579,"丽江":331,"贵港":705,"黑河":10,"安康":381,"葫芦岛":6562,"资阳":1320,"鹤壁":1904,"萍乡":173,"锡林郭勒盟":34,"防城港":33,"钦州":2141,"阿克苏":540,"鹤岗":233,"铜川":12,"喀什":68,"巴中":2540,"凉山州":0,"鹰潭":46,"石嘴山":468,"保山":152,"伊春":54,"六安":7649,"定西":75,"鸡西":2384,"临夏":1598,"海西州":0,"商洛":58,"中卫":112,"普洱":302,"湘西":4,"来宾":8,"张家界":21,"西双版纳州":0,"恩施":2215,"攀枝花":87,"晋城":6613,"乌海":361,"七台河":172,"雅安":1152,"黔西南":391,"嘉峪关":37,"淮南":6767,"潜江":0,"辽源":163,"临沧":89,"塔城":1,"克拉玛依":121,"金昌":10,"儋州":33,"海东":106,"大兴安岭":5,"兴安盟":972,"和田":0,"吐鲁番":107,"迪庆州":14,"安顺":1939,"日喀则":32,"文昌":2,"林芝":9,"保亭":0,"陵水":0,"阿勒泰":177,"天门":370,"崇左":560,"莆田":12067,"白城":2422,"铜仁":2209,"河池":1926,"辽阳":4615,"昭通":1995,"哈密":2592,"本溪":4452,"武威":1885,"德宏州":1654,"大连":18125,"成都":17618,"博尔塔拉":2094,"贺州":2168,"池州":6831,"固原":4445,"阜新":6345,"鞍山":15277,"巴音郭楞":7481,"铜陵":9225,"泰州":26770,"南通":26075,"连云港":30835,"西安":28834,"淮北":16917,"北京":35728,"南京":32406,"无锡":48220,"广州":50209,"天津":63311,"上海":160627,"常州":155553,"重庆":221058]
ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d->
def count = countMap.get(d['A'])
d['R'] = count
})
println "完成"
//def countMap = ["深圳":5,"中山":409,"贵阳":372,"南宁":1130,"台州":1325,"杭州":1177,"南昌":1369,"厦门":1370,"绍兴":1363,"徐州":1468,"长沙":1620,"东莞":1673,"郑州":1769,"遵义":1004,"温州":2012,"烟台":2080,"哈尔滨":5459,"湛江":392,"株洲":1068,"武汉":2486,"长春":2857,"岳阳":356,"肇庆":607,"镇江":1043,"芜湖":402,"银川":505,"合肥":2818,"宜昌":786,"淄博":2441,"佛山":3067,"马鞍山":227,"清远":537,"汕头":707,"洛阳":552,"济南":3456,"九江":138,"秦皇岛":1495,"宁波":3772,"乌鲁木齐":2273,"沧州":394,"泉州":4692,"廊坊":1120,"唐山":2626,"石家庄":4914,"漳州":4071,"张家口":357,"沈阳":5000,"青岛":5052,"柳州":2270,"安庆":756,"兰州":5399,"拉萨":784,"昆明":5436,"吉林":2274,"惠州":6206,"嘉兴":6142,"常德":963,"梅州":317,"绵阳":494,"海口":7883,"江门":663,"临沂":1473,"日照":2070,"襄阳":209,"湖州":3667,"平顶山":1501,"泰安":356,"大同":2814,"衡阳":241,"桂林":372,"临汾":372,"锦州":2875,"北海":2507,"赣州":1659,"商丘":644,"德州":639,"渭南":430,"咸阳":301,"聊城":932,"保定":3672,"许昌":266,"邢台":277,"潍坊":3743,"西宁":4002,"宝鸡":204,"邯郸":2361,"安阳":557,"济宁":2194,"龙岩":367,"太原":7524,"南充":568,"盐城":6512,"阜阳":198,"运城":462,"菏泽":320,"晋中":539,"新乡":768,"滁州":1025,"衡水":969,"信阳":1048,"长治":806,"韶关":696,"荆州":361,"汉中":118,"威海":3030,"孝感":984,"东营":874,"宜春":251,"开封":111,"驻马店":549,"湘潭":410,"黄冈":250,"三亚":7481,"上饶":164,"眉山":284,"绥化":851,"宁德":1469,"苏州":8876,"焦作":767,"牡丹江":1009,"齐齐哈尔":566,"枣庄":2636,"河源":167,"宿迁":3428,"遂宁":81,"滨州":3454,"漯河":308,"周口":1670,"茂名":347,"大庆":932,"平凉":210,"南阳":4087,"黄石":99,"鄂尔多斯":452,"营口":1470,"包头":4789,"吉安":442,"金华":13539,"通辽":238,"泸州":183,"乐山":2776,"佳木斯":506,"三门峡":574,"承德":2923,"荆门":64,"曲靖":1521,"忻州":770,"潮州":359,"永州":209,"揭阳":934,"呼伦贝尔":353,"扬州":12477,"宣城":293,"赤峰":2770,"吕梁":2618,"宜宾":322,"濮阳":2389,"宿州":264,"阳江":270,"延安":388,"丹东":885,"怀化":258,"黄山":312,"十堰":2687,"抚州":179,"郴州":4355,"衢州":1714,"陇南":4,"榆林":4694,"延边":1640,"抚顺":258,"丽水":817,"庆阳":2069,"玉溪":254,"玉林":544,"咸宁":1983,"舟山":1970,"达州":990,"亳州":289,"呼和浩特":10185,"张掖":64,"广元":148,"益阳":156,"汕尾":2392,"昌吉":225,"蚌埠":5496,"天水":173,"随州":211,"珠海":13972,"巴彦淖尔":0,"楚雄":95,"乌兰察布":172,"新余":86,"南平":7781,"广安":2129,"景德镇":474,"铁岭":2676,"松原":747,"淮安":13859,"三明":6425,"朝阳":3393,"酒泉":479,"四平":261,"梧州":248,"自贡":2983,"黔东南":205,"大理":1914,"黔南":866,"内江":177,"仙桃":70,"白银":2398,"朔州":1861,"娄底":2601,"六盘水":277,"百色":73,"阳泉":2135,"德阳":9330,"邵阳":2425,"盘锦":4269,"双鸭山":0,"白山":60,"通化":2379,"红河":4013,"鄂州":149,"吴忠":244,"毕节":88,"云浮":10,"伊犁":579,"丽江":331,"贵港":705,"黑河":10,"安康":381,"葫芦岛":6562,"资阳":1320,"鹤壁":1904,"萍乡":173,"锡林郭勒盟":34,"防城港":33,"钦州":2141,"阿克苏":540,"鹤岗":233,"铜川":12,"喀什":68,"巴中":2540,"凉山州":0,"鹰潭":46,"石嘴山":468,"保山":152,"伊春":54,"六安":7649,"定西":75,"鸡西":2384,"临夏":1598,"海西州":0,"商洛":58,"中卫":112,"普洱":302,"湘西":4,"来宾":8,"张家界":21,"西双版纳州":0,"恩施":2215,"攀枝花":87,"晋城":6613,"乌海":361,"七台河":172,"雅安":1152,"黔西南":391,"嘉峪关":37,"淮南":6767,"潜江":0,"辽源":163,"临沧":89,"塔城":1,"克拉玛依":121,"金昌":10,"儋州":33,"海东":106,"大兴安岭":5,"兴安盟":972,"和田":0,"吐鲁番":107,"迪庆州":14,"安顺":1939,"日喀则":32,"文昌":2,"林芝":9,"保亭":0,"陵水":0,"阿勒泰":177,"天门":370,"崇左":560,"莆田":12067,"白城":2422,"铜仁":2209,"河池":1926,"辽阳":4615,"昭通":1995,"哈密":2592,"本溪":4452,"武威":1885,"德宏州":1654,"大连":18125,"成都":17618,"博尔塔拉":2094,"贺州":2168,"池州":6831,"固原":4445,"阜新":6345,"鞍山":15277,"巴音郭楞":7481,"铜陵":9225,"泰州":26770,"南通":26075,"连云港":30835,"西安":28834,"淮北":16917,"北京":35728,"南京":32406,"无锡":48220,"广州":50209,"天津":63311,"上海":160627,"常州":155553,"重庆":221058]
//ExcelUtils.multiThreadingModify("D:\\数据\\房源\\估值数据统计结果20220715.xlsx",{d->
// def count = countMap.get(d['A'])
// d['R'] = count
//})
//
//println "完成"
def findThirdParty(city, region, name, coll){
@@ -255,21 +287,25 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
// list << d
//}
//
//def citys = ["淮安","扬州","南京","苏州","无锡","常州","徐州","镇江","南通","泰州","宿迁","连云港","盐城"]
//def rows = []
//rows << ["区域","小区量", "别名量", "地址别名量"]
//map.each{region, list->
//rows << ["城市","小区量", "别名量", "地址别名量"]
//for(def c : citys){
// def districts = CommonDatautils.findAllDistrictByCity(c,"district",["city","region","name","alias", "addr_alias"])
// def alias_count = 0
// def addr_alias_count = 0
// for(def r : list){
// if(r.mapping_alias && r.mapping_alias.size()>0){
// for(def r : districts){
// if(r.alias && r.alias.size()>0){
// alias_count ++
// }
// if(r.addr_alias && r.addr_alias.size()>0){
// addr_alias_count++
// }
// }
//
// rows << [region, list.size(), alias_count, addr_alias_count]
// rows << [c, districts.size(), alias_count, addr_alias_count]
//}
//
//ExcelUtils.write("D:\\数据\\重庆农商行\\重庆数据统计.xlsx", rows, "楼盘统计")
//
//
//ExcelUtils.write("D:\\数据\\江苏\\江苏数据统计.xlsx", rows, "结果")
@@ -0,0 +1,31 @@
package com.fangjia.script.jiangshu
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class JsBlankTest {
static main(args) {
def task = new JsBlankTest();
task.run();
}
def run(){
def path = "D:\\Êý¾Ý\\½­ËÕ\\"
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Q'] = jsonObject.get("name")
row['R'] = jsonObject.get("id")
}
}
})
}
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202205);
def count2 = findTotalCount(source, 202206);
def count1 = findTotalCount(source, 202206);
def count2 = findTotalCount(source, 202207);
println source +" "+count1 +" "+count2
}
}