diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy new file mode 100644 index 000000000..7a641d018 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy @@ -0,0 +1,88 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql +import org.bson.types.ObjectId + +class AddThirdPartyPrice { + + public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new AddThirdPartyPrice() + task.run() + } + + def run(){ +//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关" + def citys = ["林芝"] + def rows = [] + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + for(def c : citys){ + println c +" "+cityCodeDatas.get(c) + def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c) + def rets = createDate(dists, distCodeMap) + println dists.size() +" "+rets.size() + if(rets){ + rows.addAll(rets) + } + } + + def fileName = "/home/goojia/ccb/BIGDATA_THIRD_PARTY_PRICE_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_THIRD_PARTY_PRICE_GJ.dat" + } + + def header = [] + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + } + + def createDate(dists, distCodeMap){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + + def results = [] + for(def d : dists){ + + def cityId = cityCodeDatas.get(d.city) + def data = distCodeMap.get(d.region) + def avgPrice = findGuidedPrice(cityId, d._id.toString()) + if(!avgPrice){ + def dbo = CommonDatautils.queryHouseCalutePrice(d.city, d.region, d.name, DateUtils.parseFromTightYYYYMMDD("2022-05-01")) + if(dbo){ + avgPrice = dbo.avg_price + } + } + if(avgPrice){ + def ret = [] + ret << new ObjectId().toString() + ret << cityId + ret << data.district_code + ret << d._id.toString() + ret << d.name + ret << "18" + ret << "21世纪" + ret << 202209 + ret << avgPrice + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") + results << ret + } + } + return results; + } + + + def findGuidedPrice(cityId, commid){ + def sql = "select avg_price from guided_price_gj where cityid='${cityId}' and commid='${commid}' and dealdate>=to_date('2022-01-01','yyyy-MM-dd')" + def avg_price = null + sqlInstance.eachRow(sql,{r-> + avg_price = r.avg_price + }); + return avg_price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 22ffc122b..71c378532 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -52,11 +52,11 @@ class ExportOracleAllDataTest { // //处理小区情况 // exportBigdataCommunity()// // exportCommunityStaging()//没cityid - exportCommunityBuileType()// +// exportCommunityBuileType()// exportThirdPartyPrice2(new_dists)// - exportBigdataForsale(new_dists) //2022-07-19 - exportBigdataDeal( new_dists) //2022-07-01 +// exportBigdataForsale(new_dists) //2022-07-19 +// exportBigdataDeal( new_dists) //2022-07-01 // // //第三方网站权重表 // exportWebsiteRate(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 915f57f71..f8e04a350 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -33,8 +33,8 @@ import java.util.regex.*; def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", -// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -465,11 +465,35 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // } //},0) -def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] -for(def c : citys){ - def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"]) - println c +" "+districts.size() -} +//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] +//for(def c : citys){ +// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"]) +// println c +" "+districts.size() +//} + +def cityCodeMap = CcbCommonUtils.findCityCodeValue() +ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d -> +// cityCodeMap.each {k,v-> +// if(d['A'].toString()==v){ +// d['C'] = k +// println k +// } +// } +// if(!d['D']){ +// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) +// } + d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) + +}); + +//def citys = CcbCommonUtils.citys_252 +//def runCitys = ['聊城','宝鸡','韶关','阜阳','宜春','周口','河源','忻州','三门峡','永州','衢州','盘锦','白银','玉溪','达州','景德镇','鹤壁','辽阳','六盘水','张家界','海西州','兴安盟','阿坝州','那曲','西双版纳州','辽源','邯郸','平顶山','南阳','商丘','南平','黄冈','鄂尔多斯','泸州','六安','葫芦岛','潮州','宿州','庆阳','怀化','张掖','黔东南','云浮','本溪','百色','伊春','昭通','锡林郭勒盟','克拉玛依','天门','博尔塔拉','大兴安岭','潍坊','绵阳','梅州','襄阳','包头','枣庄','东营','信阳','晋中','开封','承德','茂名','荆门','红河','陇南','铁岭','黄山','咸宁','延安','广元','乌兰察布','随州','益阳','阳泉','内江','酒泉','毕节','安顺','丽江','商洛','来宾','黔西南','防城港','铜仁','儋州','崇左','衡阳','咸阳','新乡','南充','荆州','德阳','汉中','湘潭','赤峰','佳木斯','丹东','淮南','抚州','丽水','广安','亳州','昌吉','鄂州','白城','恩施','伊犁','铜陵','林芝','海东','桂林','临汾','渭南','滨州','菏泽','孝感','营口','濮阳','抚顺','自贡','新余','鸡西','钦州','淮北','安康','资阳','乌海','中卫','鹤岗','石嘴山','日喀则','昌都','德宏州','吐鲁番','阿勒泰','临沂','威海','泰安','宿迁','德州','安阳','驻马店','绥化','上饶','莆田','蚌埠','大庆','漯河','十堰','晋城','宣城','舟山','邵阳','大理','楚雄','天水','池州','松原','阜新','巴音郭楞','仙桃','武威','普洱','河池','铜川','迪庆州','潜江','贺州','塔城','济宁','江门','赣州','许昌','滁州','龙岩','运城','长治','乐山','郴州','遂宁','曲靖','呼伦贝尔','阳江','朝阳','巴彦淖尔','朔州','娄底','梧州','巴中','黔南','吴忠','临夏','凉山州','定西','萍乡','甘南','攀枝花','克孜勒苏','山南','临沧','金昌','邢台','衡水','宁德','榆林','鞍山','焦作','牡丹江','齐齐哈尔','眉山','三明','吕梁','吉安','平凉','通辽','揭阳','黄石','宜宾','延边','汕尾','玉林','通化','四平','阿克苏','贵港','白山','双鸭山','湘西','哈密','保山','鹰潭','喀什','雅安','黑河','和田','固原','阿拉善盟','七台河','嘉峪关'] +//for(def c : citys){ +// if(!runCitys.contains(c)){ +// def code = cityCodeMap.get(c) +// println code +" "+ c +// } +//} // @@ -757,15 +781,26 @@ for(def c : citys){ // return count //} // -//def findCount(cityid, sqlInstance){ -// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; -// def count = 0 -// -// sqlInstance.eachRow(sql,{r-> -// count = r.count -// }); -// return count -//} +def findCount(cityid, sqlInstance){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + + +def findThirdCommCount(cityid, sqlInstance){ + def sql = "select count(*) as count from (select count(*) from bigdata_third_party_price_gj where price_date=202209 and cityid='${cityid}' group by commid)"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} // // // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 9fd693e39..7637891ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202207 - def endDate = 202209 + def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202202 + def endDate = 202204 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -46,10 +46,10 @@ class ThirdPartyPriceBatch extends BaseBatch { def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ def statMap = getDistrictStatByCity(city, weekend) - def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) @@ -502,17 +502,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject() q.put("city", city) -<<<<<<< .mine - - //q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) - q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) -======= - q.put("s_date", endDate) - // q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) - - ->>>>>>> .theirs def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 24d6e007e..26520d0d5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -13,7 +13,7 @@ import groovy.sql.Sql @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHeTest extends BaseBatch{ - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { runBatch(new ExportDistrictRongHeTest(), args); @@ -28,24 +28,31 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() + +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// + + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() // } SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -54,46 +61,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ } -// -// def runByCity(){ -// def citys = CcbCommonUtils.citys; -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// -// -//// //检查ok文件是否上传 -//// exportAllBigdataDealFJW() -//// for(def city : citys) { -//// def cityCode = cityCodeDataMap.get(city) -//// -// //按天导(一个文件夹下) -// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// //按天导(每天一个文件夹) -// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// -//// exportBigdataCommunityGJRH(cityCode) -//// exportBigdataBuildingFJW(cityCode) -//// exportBigdataCommfeeGj(cityCode) -//// -//// exportBigdataCommbuildstruGJ(cityCode) -//// exportBigdataCommhousetypeGJ(cityCode) -//// exportBigdataCommbuildyearGj(cityCode) -//// exportBigdataCommhouseuseGJ(cityCode) -//// exportBigdataPositionGJ(cityCode) -//// exportBigdataDistrictFJW(cityCode) -//// exportBigdataAliasFJW(cityCode) -//// -//// exportBigdataBuildprojFjw(cityCode) -//// exportBigdataCommlayoutpicFjw(cityCode) -//// } -//// exportEvaluateRate() -// -//// -// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// -// exportOk() -// } + def exportBigdataCommunityGJRH(){ @@ -773,6 +741,145 @@ class ExportDistrictRongHeTest extends BaseBatch{ println "BIGDATA_BUILDING_FJW 上传Ftp完成......" } + def exportBigdataCityPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_CITYPRICE 上传Ftp完成......" + } + + + def exportBigdataDistPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.DISTID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DISTPRICE 上传Ftp完成......" + } + + + def exportBigdataCommPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.COMMID + data << r.SALEPRICE?r.SALEPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.HOUSEUSE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.DATATYPE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMPRICE 上传Ftp完成......" + } + def exportEvaluateRate(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8f53bcb0a..49bacf4ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1672,6 +1672,22 @@ class CommonDatautils { } + public static def queryHouseCalutePrice(city, region, name, startWeekendDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + queryCondition.put('weekend', new BasicDBObject('$gte', startWeekendDate)); + queryCondition.put("value_type", new BasicDBObject('$in', [1,2])); + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(1) + def dbo = null; + while(cur.hasNext()) { + dbo = cur.next() + } + return dbo; + } + + public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 6ccaabde3..6d692218b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,13 +48,13 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-09-01" + static def statDate = "2022-10-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es -// task.esfhouseToSaveEs();//二手房保存到es + task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index ca99d1f97..f033823f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -398,13 +398,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 30772c368..0ca1bc62a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index e76898c7f..5c602f855 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "安居客"// - def citys = findCrawlCity(source, 202208); + def source = "我爱我家"// + def citys = findCrawlCity(source, 202209); for(def city : citys){ - def count1 = findCount(source, city, 202208); - def count2 = findCount(source, city, 202209); + def count1 = findCount(source, city, 202209); + def count2 = findCount(source, city, 202210); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 57c47a013..78f841680 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -24,8 +24,9 @@ class SynchDistrictPrice extends BaseBatch{ // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" - def esSources = [ "搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] + // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", + // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" + def esSources = ["贝壳找房":"贝壳找房","链家":"链家","麦田":"麦田"] static main(args) {