Merge branch 'script_gj' into 'master'

Script gj

See merge request !459
This commit is contained in:
2022-11-21 17:55:49 +08:00
9 changed files with 90 additions and 59 deletions
@@ -36,7 +36,7 @@ class BigdataBuildingBatch extends BaseBatch{
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap)
println city + " " + rets.size() println city + " " + rets.size()
def fileName = "/home/goojia/ccb/bigdata_building_fjw.dat" def fileName = "/ccb/bigdata_building_fjw.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat" fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat"
} }
@@ -63,7 +63,7 @@ class BigdataDealBatch extends BaseBatch{
curDate = args[1].toString().toInteger() curDate = args[1].toString().toInteger()
} }
println "curDate:"+curDate // println "curDate:"+curDate
// for(def city : citys){ // for(def city : citys){
//// if(!ccbCitys.contains(city)){ //// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city); // addDealPriceMatch(city);
@@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{
} }
def trades = getCityData(city, statMap, ppeDists) // def trades = getCityData(city, statMap, ppeDists)
if(trades){ // if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") // def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets) // rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size() // println "【${index}】"+city +" 成交共:"+rets.size()
//
def cityDatas = [] // def cityDatas = []
cityDatas << headers // cityDatas << headers
cityDatas.addAll(rows) // cityDatas.addAll(rows)
} // }
} }
@@ -105,7 +105,7 @@ class BigdataDealBatch extends BaseBatch{
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
} }
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size() println "txt 生成 " + rows.size()
} }
@@ -114,9 +114,11 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size()) final CountDownLatch latch = new CountDownLatch(rets.size())
def lianjiaPriceMap = findThirdPartyPrice("上海") // def lianjiaPriceMap = findThirdPartyPrice("上海")
def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') // def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) // def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
def ccbPriceMap = getCommunityPrices("上海")
for(def i=0;i<rets.size();i++){ for(def i=0;i<rets.size();i++){
final def trade = rets[i] final def trade = rets[i]
fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
@@ -125,7 +127,7 @@ class BigdataDealBatch extends BaseBatch{
totalCount.addAndGet(1) totalCount.addAndGet(1)
if(source == "youda"){ if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, lianjiaPriceMap) def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
if(ret){ if(ret){
result << ret result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString() println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
@@ -151,6 +153,21 @@ class BigdataDealBatch extends BaseBatch{
return result return result
} }
def getCommunityPrices(def city){
def sql = "select commid, avg_price from community_base_price_gj where year_month=${endDate} and cityname='${city}'";
def map = [:]
try{
sqlInstance.eachRow(sql,{
map.put(it.commid, it.avg_price)
});
}catch(Exception e){
getPpeGTE15Datas(city)
}
return map
}
def findThirdPartyPrice(def city) { def findThirdPartyPrice(def city) {
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) q.put("city", city)
@@ -174,13 +191,13 @@ class BigdataDealBatch extends BaseBatch{
return map return map
} }
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def lianjiaPriceMap){ def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def ccbPriceMap){
def ret = [] def ret = []
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){ if(!obj){
return ret; return ret;
} }
def price = lianjiaPriceMap.get(obj.region+"##"+obj.name) def price = ccbPriceMap.get(obj._id.toString())
if(price){ if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price) def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){ if(mod<=10){
@@ -617,10 +634,10 @@ class BigdataDealBatch extends BaseBatch{
def addDealPriceMatch(city){ def addDealPriceMatch(city){
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
final CountDownLatch latch = new CountDownLatch(trades.size()) final CountDownLatch latch = new CountDownLatch(trades.size())
@@ -562,4 +562,9 @@ class CcbCommonUtils {
return maxSeqno return maxSeqno
} }
public static def deleteGuidedPrice(def dealdate){
def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
}
} }
@@ -30,6 +30,8 @@ class GuiDingPriceBatch extends BaseBatch{
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
static main(args) { static main(args) {
runBatch(new GuiDingPriceBatch(), args); runBatch(new GuiDingPriceBatch(), args);
} }
@@ -39,12 +41,13 @@ class GuiDingPriceBatch extends BaseBatch{
def citys = CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
def rows = [] def rows = []
CcbCommonUtils.deleteGuidedPrice(curDate)
for (def city : citys) { for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-11") def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -150,7 +153,7 @@ class GuiDingPriceBatch extends BaseBatch{
ret << CcbCommonUtils.getCcbCity(dist.city) ret << CcbCommonUtils.getCcbCity(dist.city)
ret << areaData.district_code ret << areaData.district_code
ret << areaData.district ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value ret << ps.guiding_value
ret << "房价网" ret << "房价网"
ret << "" ret << ""
@@ -447,23 +447,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
////},0) ////},0)
// //
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\11.11\\删除小区(2).xlsx",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
// if(d['A']){ if(d['A']){
// def dbo = CommonDatautils.findDistrictById(d['C']) def dbo = CommonDatautils.findDistrictById(d['D'])
// if(dbo){ if(dbo){
// def udata = new BasicDBObject() def udata = new BasicDBObject()
// udata.put("name_cat", 11) // udata.put("name_cat", 11)
// udata.put("status", 1) udata.put("status", 1)
// udata.put("m_date",new Date()) udata.put("m_date",new Date())
//
// def q = new BasicDBObject() def q = new BasicDBObject()
// q.put("_id", dbo._id) q.put("_id", dbo._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{ }else{
// println dbo._id.toString() +" 没找到" println dbo._id.toString() +" 没找到"
// } }
// } }
//},0) },0)
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){ //for(def c : citys){
@@ -471,20 +471,20 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
// println c +" "+districts.size() // println c +" "+districts.size()
//} //}
def cityCodeMap = CcbCommonUtils.findCityCodeValue() //def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d -> //ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d ->
// cityCodeMap.each {k,v-> //// cityCodeMap.each {k,v->
// if(d['A'].toString()==v){ //// if(d['A'].toString()==v){
// d['C'] = k //// d['C'] = k
// println k //// println k
// } //// }
// } //// }
// if(!d['D']){ //// if(!d['D']){
// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) //// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94)
// } //// }
d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) // d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94)
//
}); //});
//def citys = CcbCommonUtils.citys_252 //def citys = CcbCommonUtils.citys_252
//def runCitys = ['聊城','宝鸡','韶关','阜阳','宜春','周口','河源','忻州','三门峡','永州','衢州','盘锦','白银','玉溪','达州','景德镇','鹤壁','辽阳','六盘水','张家界','海西州','兴安盟','阿坝州','那曲','西双版纳州','辽源','邯郸','平顶山','南阳','商丘','南平','黄冈','鄂尔多斯','泸州','六安','葫芦岛','潮州','宿州','庆阳','怀化','张掖','黔东南','云浮','本溪','百色','伊春','昭通','锡林郭勒盟','克拉玛依','天门','博尔塔拉','大兴安岭','潍坊','绵阳','梅州','襄阳','包头','枣庄','东营','信阳','晋中','开封','承德','茂名','荆门','红河','陇南','铁岭','黄山','咸宁','延安','广元','乌兰察布','随州','益阳','阳泉','内江','酒泉','毕节','安顺','丽江','商洛','来宾','黔西南','防城港','铜仁','儋州','崇左','衡阳','咸阳','新乡','南充','荆州','德阳','汉中','湘潭','赤峰','佳木斯','丹东','淮南','抚州','丽水','广安','亳州','昌吉','鄂州','白城','恩施','伊犁','铜陵','林芝','海东','桂林','临汾','渭南','滨州','菏泽','孝感','营口','濮阳','抚顺','自贡','新余','鸡西','钦州','淮北','安康','资阳','乌海','中卫','鹤岗','石嘴山','日喀则','昌都','德宏州','吐鲁番','阿勒泰','临沂','威海','泰安','宿迁','德州','安阳','驻马店','绥化','上饶','莆田','蚌埠','大庆','漯河','十堰','晋城','宣城','舟山','邵阳','大理','楚雄','天水','池州','松原','阜新','巴音郭楞','仙桃','武威','普洱','河池','铜川','迪庆州','潜江','贺州','塔城','济宁','江门','赣州','许昌','滁州','龙岩','运城','长治','乐山','郴州','遂宁','曲靖','呼伦贝尔','阳江','朝阳','巴彦淖尔','朔州','娄底','梧州','巴中','黔南','吴忠','临夏','凉山州','定西','萍乡','甘南','攀枝花','克孜勒苏','山南','临沧','金昌','邢台','衡水','宁德','榆林','鞍山','焦作','牡丹江','齐齐哈尔','眉山','三明','吕梁','吉安','平凉','通辽','揭阳','黄石','宜宾','延边','汕尾','玉林','通化','四平','阿克苏','贵港','白山','双鸭山','湘西','哈密','保山','鹰潭','喀什','雅安','黑河','和田','固原','阿拉善盟','七台河','嘉峪关'] //def runCitys = ['聊城','宝鸡','韶关','阜阳','宜春','周口','河源','忻州','三门峡','永州','衢州','盘锦','白银','玉溪','达州','景德镇','鹤壁','辽阳','六盘水','张家界','海西州','兴安盟','阿坝州','那曲','西双版纳州','辽源','邯郸','平顶山','南阳','商丘','南平','黄冈','鄂尔多斯','泸州','六安','葫芦岛','潮州','宿州','庆阳','怀化','张掖','黔东南','云浮','本溪','百色','伊春','昭通','锡林郭勒盟','克拉玛依','天门','博尔塔拉','大兴安岭','潍坊','绵阳','梅州','襄阳','包头','枣庄','东营','信阳','晋中','开封','承德','茂名','荆门','红河','陇南','铁岭','黄山','咸宁','延安','广元','乌兰察布','随州','益阳','阳泉','内江','酒泉','毕节','安顺','丽江','商洛','来宾','黔西南','防城港','铜仁','儋州','崇左','衡阳','咸阳','新乡','南充','荆州','德阳','汉中','湘潭','赤峰','佳木斯','丹东','淮南','抚州','丽水','广安','亳州','昌吉','鄂州','白城','恩施','伊犁','铜陵','林芝','海东','桂林','临汾','渭南','滨州','菏泽','孝感','营口','濮阳','抚顺','自贡','新余','鸡西','钦州','淮北','安康','资阳','乌海','中卫','鹤岗','石嘴山','日喀则','昌都','德宏州','吐鲁番','阿勒泰','临沂','威海','泰安','宿迁','德州','安阳','驻马店','绥化','上饶','莆田','蚌埠','大庆','漯河','十堰','晋城','宣城','舟山','邵阳','大理','楚雄','天水','池州','松原','阜新','巴音郭楞','仙桃','武威','普洱','河池','铜川','迪庆州','潜江','贺州','塔城','济宁','江门','赣州','许昌','滁州','龙岩','运城','长治','乐山','郴州','遂宁','曲靖','呼伦贝尔','阳江','朝阳','巴彦淖尔','朔州','娄底','梧州','巴中','黔南','吴忠','临夏','凉山州','定西','萍乡','甘南','攀枝花','克孜勒苏','山南','临沧','金昌','邢台','衡水','宁德','榆林','鞍山','焦作','牡丹江','齐齐哈尔','眉山','三明','吕梁','吉安','平凉','通辽','揭阳','黄石','宜宾','延边','汕尾','玉林','通化','四平','阿克苏','贵港','白山','双鸭山','湘西','哈密','保山','鹰潭','喀什','雅安','黑河','和田','固原','阿拉善盟','七台河','嘉峪关']
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
} }
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
} }
@@ -32,8 +32,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", def sqlInstance = null
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def batchsize = 1000 public static def batchsize = 1000
static main(args) { static main(args) {
@@ -43,6 +42,13 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
if(citys.size()==92){
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else{
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
def rows = [] def rows = []
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for (def city : citys) { // for (def city : citys) {
@@ -183,7 +183,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href") def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text() def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){ if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl detailUrls << durl
parse(city, e, region) parse(city, e, region)
@@ -251,7 +251,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href") def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text() def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){ if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl detailUrls << durl
parse(city, e, regionText) parse(city, e, regionText)