成交修改

This commit is contained in:
2022-12-21 18:04:47 +08:00
parent 9743a72aeb
commit 73f4ebbd5c
13 changed files with 518 additions and 145 deletions
@@ -58,17 +58,17 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) { public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys; // def citys = CcbCommonUtils.citys;
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){ if(args && args.size()>=2){
curDate = args[1].toString().toInteger() curDate = args[1].toString().toInteger()
} }
// println "curDate:"+curDate println "curDate:"+curDate
// for(def city : citys){ for(def city : citys){
//// if(!ccbCitys.contains(city)){ // if(!ccbCitys.contains(city)){
// addDealPriceMatch(city); addDealPriceMatch(city);
//// } // }
// } }
def rows = [] def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -86,17 +86,16 @@ class BigdataDealBatch extends BaseBatch{
println city +" "+youdaDatas.size() println city +" "+youdaDatas.size()
} }
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
// def trades = getCityData(city, statMap, ppeDists) def cityDatas = []
// if(trades){ cityDatas << headers
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") cityDatas.addAll(rows)
// rows.addAll(rets) }
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
} }
@@ -105,7 +104,7 @@ class BigdataDealBatch extends BaseBatch{
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
} }
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size() println "txt 生成 " + rows.size()
} }
@@ -75,7 +75,7 @@ class FjwPricePkCcbPrice {
def cityId = d['D'] def cityId = d['D']
def dist = CommonDatautils.findDistrictById(d['B']) def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){ if(dist){
def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202211) def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209)
if(map){ if(map){
d['O'] = map.avg_price d['O'] = map.avg_price
d['P'] = map.calc_type d['P'] = map.calc_type
@@ -38,16 +38,16 @@ class GuiDingPriceBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
def rows = [] def rows = []
// CcbCommonUtils.deleteGuidedPrice(curDate) CcbCommonUtils.deleteGuidedPrice(curDate)
for (def city : citys) { for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25") def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-25", "2022-12-19")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
@@ -47,41 +47,41 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水'] //def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
//
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() //def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def rets = [] //def rets = []
for(def c : citys){ //for(def c : citys){
def citycode = cityCodeDataMap.get(c) // def citycode = cityCodeDataMap.get(c)
def datas = findAllData(sqlInstance236, citycode) // def datas = findAllData(sqlInstance236, citycode)
def totalCount = datas.size() // def totalCount = datas.size()
def landareaCount = 0; // def landareaCount = 0;
def buildareaCount = 0 // def buildareaCount = 0
def greeningrateCount = 0 // def greeningrateCount = 0
def plotratioCount = 0 // def plotratioCount = 0
for(def m : datas){ // for(def m : datas){
if(m.landarea){ // if(m.landarea){
landareaCount ++ // landareaCount ++
} // }
if(m.buildarea){ // if(m.buildarea){
buildareaCount++ // buildareaCount++
} // }
if(m.greeningrate){ // if(m.greeningrate){
greeningrateCount++ // greeningrateCount++
} // }
if(m.plotratio){ // if(m.plotratio){
plotratioCount++ // plotratioCount++
} // }
} // }
//
def landAreaPre = landareaCount*100/totalCount // def landAreaPre = landareaCount*100/totalCount
def buildAreaPre = buildareaCount*100/totalCount // def buildAreaPre = buildareaCount*100/totalCount
def greeningratePre = greeningrateCount*100/totalCount // def greeningratePre = greeningrateCount*100/totalCount
def plotratioPre = plotratioCount*100/totalCount // def plotratioPre = plotratioCount*100/totalCount
rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre] // rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre]
} //}
//
ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率") //ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率")
def findAllData(sqlInstance236, citycode){ def findAllData(sqlInstance236, citycode){
def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'" def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'"
@@ -1149,6 +1149,30 @@ def findThirdCommCount(cityid, sqlInstance){
ExcelUtils.modify("D:\\数据\\建行\\二期\\付.xlsx",{d->
def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district")
if(obj){
def count = findGuidedPrice(sqlInstance236, obj._id.toString())
if(count){
d['M'] = "存在"
}
d['N'] = obj.name_cat
d['O'] = obj.estate_type2
}
})
def findGuidedPrice(sqlInstance, id){
def count = 0
def sql = "select count(*) as count from guided_price_gj where commid='${id}'"
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
@@ -50,7 +50,7 @@ class TradeStatLog {
// task.deleteDealData2() // task.deleteDealData2()
// task.statTableCount() // task.statTableCount()
// task.filterTrade() // task.filterTrade()
task.updateDealPriceType() // task.updateDealPriceType()
} }
@@ -90,15 +90,15 @@ class TradeStatLog {
} }
def run(){ def run(){
def citys = CcbCommonUtils.citys_92; def citys = CcbCommonUtils.citys_all;
def rets = [] def rets = []
rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01" //def startDate = "2021-05-01"
def startDate = "2022-08-01" def startDate = "2022-09-01"
def date = "2022-09-01" def date = "2022-10-01"
def endDate = "2022-10-01" def endDate = "2022-11-01"
def newDate = "2022-11-01" def newDate = "2022-12-01"
for(def city : citys){ for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -5,9 +5,36 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils import com.fangjia.script.gj.EsUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.io.FileUtils import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId import org.bson.types.ObjectId
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
def rets = []
for(def d : datas){
def arrs = d.split("@")
rets << ["commId":arrs[0], "avg_price":arrs[4]]
}
for(def r : rets){
if(r.avg_price){
def udata = new BasicDBObject()
udata.put("date", 202210)
udata.put("avg", r.avg_price.toInteger())
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
}else{
println r.commId +" 不存在"
}
}
//List<String> list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv")) //List<String> list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv"))
//println "过滤前:"+list.size()+ " "+new Date().toLocaleString() //println "过滤前:"+list.size()+ " "+new Date().toLocaleString()
//def ret = [] //def ret = []
@@ -48,29 +75,29 @@ import org.bson.types.ObjectId
// return location; // return location;
//} //}
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv")) //List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
def map1 = [:] //def map1 = [:]
for(def s : list1){ //for(def s : list1){
def row = s.split("@\\|@") // def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4] // def key = row[2]+"##"+row[3]+"##"+row[4]
map1.put(key, row[5]) // map1.put(key, row[5])
} //}
//
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) //List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
def map2 = [:] //def map2 = [:]
for(def s : list2) { //for(def s : list2) {
def row = s.split("@\\|@") // def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4] // def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row[5]) // map2.put(key, row[5])
} //}
def rows = [] //def rows = []
map1.each {k,v-> //map1.each {k,v->
def row = [] // def row = []
def keys = k.split("##") // def keys = k.split("##")
def lastPrice = map2.get(k) // def lastPrice = map2.get(k)
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] // rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
} //}
ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据") //ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d-> //ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd")) // def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
// if(obj){ // if(obj){
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息 //福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202210.csv") def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv")
def count0 = 0 def count0 = 0
def count1 = 0 def count1 = 0
def count2 = 0 def count2 = 0
File diff suppressed because one or more lines are too long
@@ -52,7 +52,7 @@ class YouDaDataParser {
static main(args) { static main(args) {
def task = new YouDaDataParser(); def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo // task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo // task.esfParser("D:\\数据\\友达\\2022年12月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es // task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles) { for(def e : cityEles) {
def href = e.attr("href") def href = e.attr("href")
if (!href.contains("fang.lianjia")){ if (!href.contains("fang.lianjia") ){
cityMap.put(e.text(), href) cityMap.put(e.text(), href)
} }
} }
@@ -247,6 +247,35 @@ class GaoDeUtils {
return Collections.emptyList() return Collections.emptyList()
} }
static List<AmapPoi> getAmapById2(id){
def params = [:]
params.id = id
params.output = 'json'
params.key = "66e6771767ab293ace441553c36621c1"
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params);
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoi> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
System.exit(1)
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
return Collections.emptyList()
}
/** /**
* 获取详情 * 获取详情
*/ */
@@ -40,11 +40,11 @@ class DataStat {
def run(){ def run(){
def source = "链家"// def source = "365淘房"//
def citys = findCrawlCity(source, 202209); def citys = findCrawlCity(source, 202210);
for(def city : citys){ for(def city : citys){
def count1 = findCount(source, city, 202209); def count1 = findCount(source, city, 202210);
def count2 = findCount(source, city, 202210); def count2 = findCount(source, city, 202211);
println city +" "+count1+" "+count2 println city +" "+count1+" "+count2
// println count1 // println count1
// println "--- ----" // println "--- ----"
@@ -65,6 +65,8 @@ class MongoUtils {
if (domainUrl == MongoDB_Domain_mongo) { if (domainUrl == MongoDB_Domain_mongo) {
port == 28017; port == 28017;
} }
if ((domainUrl == "192.168.0.136") && (port == 27017 || port == 28017)) { if ((domainUrl == "192.168.0.136") && (port == 27017 || port == 28017)) {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name) return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
} }