Merge branch 'script_gj' into 'master'
补充15个城市的第三方 See merge request !451
This commit is contained in:
+88
@@ -0,0 +1,88 @@
|
|||||||
|
package com.fangjia.script.ccbevaluation
|
||||||
|
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.gj.ccb.FileUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import groovy.sql.Sql
|
||||||
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
|
class AddThirdPartyPrice {
|
||||||
|
|
||||||
|
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||||
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new AddThirdPartyPrice()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"
|
||||||
|
def citys = ["林芝"]
|
||||||
|
def rows = []
|
||||||
|
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||||
|
for(def c : citys){
|
||||||
|
println c +" "+cityCodeDatas.get(c)
|
||||||
|
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||||
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c)
|
||||||
|
def rets = createDate(dists, distCodeMap)
|
||||||
|
println dists.size() +" "+rets.size()
|
||||||
|
if(rets){
|
||||||
|
rows.addAll(rets)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def fileName = "/home/goojia/ccb/BIGDATA_THIRD_PARTY_PRICE_GJ.dat"
|
||||||
|
if(CommonDatautils.isWindow()){
|
||||||
|
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_THIRD_PARTY_PRICE_GJ.dat"
|
||||||
|
}
|
||||||
|
|
||||||
|
def header = []
|
||||||
|
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||||
|
}
|
||||||
|
|
||||||
|
def createDate(dists, distCodeMap){
|
||||||
|
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||||
|
|
||||||
|
def results = []
|
||||||
|
for(def d : dists){
|
||||||
|
|
||||||
|
def cityId = cityCodeDatas.get(d.city)
|
||||||
|
def data = distCodeMap.get(d.region)
|
||||||
|
def avgPrice = findGuidedPrice(cityId, d._id.toString())
|
||||||
|
if(!avgPrice){
|
||||||
|
def dbo = CommonDatautils.queryHouseCalutePrice(d.city, d.region, d.name, DateUtils.parseFromTightYYYYMMDD("2022-05-01"))
|
||||||
|
if(dbo){
|
||||||
|
avgPrice = dbo.avg_price
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(avgPrice){
|
||||||
|
def ret = []
|
||||||
|
ret << new ObjectId().toString()
|
||||||
|
ret << cityId
|
||||||
|
ret << data.district_code
|
||||||
|
ret << d._id.toString()
|
||||||
|
ret << d.name
|
||||||
|
ret << "18"
|
||||||
|
ret << "21世纪"
|
||||||
|
ret << 202209
|
||||||
|
ret << avgPrice
|
||||||
|
ret << ""
|
||||||
|
ret << ""
|
||||||
|
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||||
|
results << ret
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findGuidedPrice(cityId, commid){
|
||||||
|
def sql = "select avg_price from guided_price_gj where cityid='${cityId}' and commid='${commid}' and dealdate>=to_date('2022-01-01','yyyy-MM-dd')"
|
||||||
|
def avg_price = null
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
avg_price = r.avg_price
|
||||||
|
});
|
||||||
|
return avg_price
|
||||||
|
}
|
||||||
|
}
|
||||||
+3
-3
@@ -52,11 +52,11 @@ class ExportOracleAllDataTest {
|
|||||||
// //处理小区情况
|
// //处理小区情况
|
||||||
// exportBigdataCommunity()//
|
// exportBigdataCommunity()//
|
||||||
// exportCommunityStaging()//没cityid
|
// exportCommunityStaging()//没cityid
|
||||||
exportCommunityBuileType()//
|
// exportCommunityBuileType()//
|
||||||
|
|
||||||
exportThirdPartyPrice2(new_dists)//
|
exportThirdPartyPrice2(new_dists)//
|
||||||
exportBigdataForsale(new_dists) //2022-07-19
|
// exportBigdataForsale(new_dists) //2022-07-19
|
||||||
exportBigdataDeal( new_dists) //2022-07-01
|
// exportBigdataDeal( new_dists) //2022-07-01
|
||||||
//
|
//
|
||||||
// //第三方网站权重表
|
// //第三方网站权重表
|
||||||
// exportWebsiteRate();
|
// exportWebsiteRate();
|
||||||
|
|||||||
+51
-16
@@ -33,8 +33,8 @@ import java.util.regex.*;
|
|||||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
@@ -465,11 +465,35 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
|||||||
// }
|
// }
|
||||||
//},0)
|
//},0)
|
||||||
|
|
||||||
def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
||||||
for(def c : citys){
|
//for(def c : citys){
|
||||||
def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
|
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
|
||||||
println c +" "+districts.size()
|
// println c +" "+districts.size()
|
||||||
}
|
//}
|
||||||
|
|
||||||
|
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||||
|
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d ->
|
||||||
|
// cityCodeMap.each {k,v->
|
||||||
|
// if(d['A'].toString()==v){
|
||||||
|
// d['C'] = k
|
||||||
|
// println k
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// if(!d['D']){
|
||||||
|
// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94)
|
||||||
|
// }
|
||||||
|
d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94)
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
//def citys = CcbCommonUtils.citys_252
|
||||||
|
//def runCitys = ['聊城','宝鸡','韶关','阜阳','宜春','周口','河源','忻州','三门峡','永州','衢州','盘锦','白银','玉溪','达州','景德镇','鹤壁','辽阳','六盘水','张家界','海西州','兴安盟','阿坝州','那曲','西双版纳州','辽源','邯郸','平顶山','南阳','商丘','南平','黄冈','鄂尔多斯','泸州','六安','葫芦岛','潮州','宿州','庆阳','怀化','张掖','黔东南','云浮','本溪','百色','伊春','昭通','锡林郭勒盟','克拉玛依','天门','博尔塔拉','大兴安岭','潍坊','绵阳','梅州','襄阳','包头','枣庄','东营','信阳','晋中','开封','承德','茂名','荆门','红河','陇南','铁岭','黄山','咸宁','延安','广元','乌兰察布','随州','益阳','阳泉','内江','酒泉','毕节','安顺','丽江','商洛','来宾','黔西南','防城港','铜仁','儋州','崇左','衡阳','咸阳','新乡','南充','荆州','德阳','汉中','湘潭','赤峰','佳木斯','丹东','淮南','抚州','丽水','广安','亳州','昌吉','鄂州','白城','恩施','伊犁','铜陵','林芝','海东','桂林','临汾','渭南','滨州','菏泽','孝感','营口','濮阳','抚顺','自贡','新余','鸡西','钦州','淮北','安康','资阳','乌海','中卫','鹤岗','石嘴山','日喀则','昌都','德宏州','吐鲁番','阿勒泰','临沂','威海','泰安','宿迁','德州','安阳','驻马店','绥化','上饶','莆田','蚌埠','大庆','漯河','十堰','晋城','宣城','舟山','邵阳','大理','楚雄','天水','池州','松原','阜新','巴音郭楞','仙桃','武威','普洱','河池','铜川','迪庆州','潜江','贺州','塔城','济宁','江门','赣州','许昌','滁州','龙岩','运城','长治','乐山','郴州','遂宁','曲靖','呼伦贝尔','阳江','朝阳','巴彦淖尔','朔州','娄底','梧州','巴中','黔南','吴忠','临夏','凉山州','定西','萍乡','甘南','攀枝花','克孜勒苏','山南','临沧','金昌','邢台','衡水','宁德','榆林','鞍山','焦作','牡丹江','齐齐哈尔','眉山','三明','吕梁','吉安','平凉','通辽','揭阳','黄石','宜宾','延边','汕尾','玉林','通化','四平','阿克苏','贵港','白山','双鸭山','湘西','哈密','保山','鹰潭','喀什','雅安','黑河','和田','固原','阿拉善盟','七台河','嘉峪关']
|
||||||
|
//for(def c : citys){
|
||||||
|
// if(!runCitys.contains(c)){
|
||||||
|
// def code = cityCodeMap.get(c)
|
||||||
|
// println code +" "+ c
|
||||||
|
// }
|
||||||
|
//}
|
||||||
|
|
||||||
|
|
||||||
//
|
//
|
||||||
@@ -757,15 +781,26 @@ for(def c : citys){
|
|||||||
// return count
|
// return count
|
||||||
//}
|
//}
|
||||||
//
|
//
|
||||||
//def findCount(cityid, sqlInstance){
|
def findCount(cityid, sqlInstance){
|
||||||
// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0";
|
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0";
|
||||||
// def count = 0
|
def count = 0
|
||||||
//
|
|
||||||
// sqlInstance.eachRow(sql,{r->
|
sqlInstance.eachRow(sql,{r->
|
||||||
// count = r.count
|
count = r.count
|
||||||
// });
|
});
|
||||||
// return count
|
return count
|
||||||
//}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findThirdCommCount(cityid, sqlInstance){
|
||||||
|
def sql = "select count(*) as count from (select count(*) from bigdata_third_party_price_gj where price_date=202209 and cityid='${cityid}' group by commid)";
|
||||||
|
def count = 0
|
||||||
|
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
count = r.count
|
||||||
|
});
|
||||||
|
return count
|
||||||
|
}
|
||||||
//
|
//
|
||||||
//
|
//
|
||||||
//
|
//
|
||||||
|
|||||||
+4
-14
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
|||||||
@BatchOwner(Watcher.GaoJuan)
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
class ThirdPartyPriceBatch extends BaseBatch {
|
class ThirdPartyPriceBatch extends BaseBatch {
|
||||||
|
|
||||||
def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
def startDate = 202207
|
def startDate = 202202
|
||||||
def endDate = 202209
|
def endDate = 202204
|
||||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||||
|
|
||||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||||
@@ -46,10 +46,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
|||||||
def rows = []
|
def rows = []
|
||||||
def index = 0
|
def index = 0
|
||||||
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||||
|
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
index++
|
index++
|
||||||
def statMap = getDistrictStatByCity(city, weekend)
|
def statMap = getDistrictStatByCity(city, weekend)
|
||||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
|
|
||||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||||
@@ -502,17 +502,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
|||||||
def findThirdPartyPrice(def city, def statMap){
|
def findThirdPartyPrice(def city, def statMap){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
<<<<<<< .mine
|
|
||||||
|
|
||||||
//q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
|
||||||
|
|
||||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||||
=======
|
|
||||||
q.put("s_date", endDate)
|
|
||||||
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
|
||||||
|
|
||||||
|
|
||||||
>>>>>>> .theirs
|
|
||||||
|
|
||||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||||
def map = [:]
|
def map = [:]
|
||||||
|
|||||||
+162
-55
@@ -13,7 +13,7 @@ import groovy.sql.Sql
|
|||||||
@BatchOwner(Watcher.GaoJuan)
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
class ExportDistrictRongHeTest extends BaseBatch{
|
class ExportDistrictRongHeTest extends BaseBatch{
|
||||||
|
|
||||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
static main(args) {
|
static main(args) {
|
||||||
runBatch(new ExportDistrictRongHeTest(), args);
|
runBatch(new ExportDistrictRongHeTest(), args);
|
||||||
@@ -28,24 +28,31 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
|||||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||||
def new_dists = []//["350604","350605","410307","410308","450181"]
|
def new_dists = []//["350604","350605","410307","410308","450181"]
|
||||||
//按天导(每天一个文件夹)
|
//按天导(每天一个文件夹)
|
||||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
|
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
|
||||||
|
|
||||||
// if(curDate.toString().endsWith("10")){
|
// if(curDate.toString().endsWith("10")){
|
||||||
exportBigdataCommunityGJRH()//
|
|
||||||
exportBigdataBuildingFJW()//
|
|
||||||
exportBigdataCommfeeGj()//
|
|
||||||
|
|
||||||
exportBigdataCommbuildstruGJ()//
|
// exportBigdataBuildprojFjw()
|
||||||
exportBigdataCommhousetypeGJ()//
|
// exportBigdataCommlayoutpicFjw()
|
||||||
exportBigdataCommbuildyearGj()//
|
// exportEvaluateRate()
|
||||||
exportBigdataCommhouseuseGJ()//
|
|
||||||
exportBigdataPositionGJ()//
|
// exportBigdataCommunityGJRH()//
|
||||||
exportBigdataDistrictFJW()//
|
// exportBigdataBuildingFJW()//
|
||||||
exportBigdataAliasFJW()//
|
// exportBigdataCommfeeGj()//
|
||||||
|
//
|
||||||
|
// exportBigdataCommbuildstruGJ()//
|
||||||
|
// exportBigdataCommhousetypeGJ()//
|
||||||
|
// exportBigdataCommbuildyearGj()//
|
||||||
|
// exportBigdataCommhouseuseGJ()//
|
||||||
|
// exportBigdataPositionGJ()//
|
||||||
|
// exportBigdataDistrictFJW()//
|
||||||
|
// exportBigdataAliasFJW()//
|
||||||
|
|
||||||
|
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||||
|
exportBigdataCityPrice(queryDate)
|
||||||
|
exportBigdataDistPrice(queryDate)
|
||||||
|
exportBigdataCommPrice(queryDate)
|
||||||
|
|
||||||
exportBigdataBuildprojFjw()
|
|
||||||
exportBigdataCommlayoutpicFjw()
|
|
||||||
exportEvaluateRate()
|
|
||||||
// }
|
// }
|
||||||
SSHHelper jschCommand = new SSHHelper();
|
SSHHelper jschCommand = new SSHHelper();
|
||||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||||
@@ -54,46 +61,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
|
||||||
// def runByCity(){
|
|
||||||
// def citys = CcbCommonUtils.citys;
|
|
||||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
|
||||||
//
|
|
||||||
//
|
|
||||||
//// //检查ok文件是否上传
|
|
||||||
//// exportAllBigdataDealFJW()
|
|
||||||
//// for(def city : citys) {
|
|
||||||
//// def cityCode = cityCodeDataMap.get(city)
|
|
||||||
////
|
|
||||||
// //按天导(一个文件夹下)
|
|
||||||
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
|
||||||
// //按天导(每天一个文件夹)
|
|
||||||
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
|
||||||
//
|
|
||||||
//// exportBigdataCommunityGJRH(cityCode)
|
|
||||||
//// exportBigdataBuildingFJW(cityCode)
|
|
||||||
//// exportBigdataCommfeeGj(cityCode)
|
|
||||||
////
|
|
||||||
//// exportBigdataCommbuildstruGJ(cityCode)
|
|
||||||
//// exportBigdataCommhousetypeGJ(cityCode)
|
|
||||||
//// exportBigdataCommbuildyearGj(cityCode)
|
|
||||||
//// exportBigdataCommhouseuseGJ(cityCode)
|
|
||||||
//// exportBigdataPositionGJ(cityCode)
|
|
||||||
//// exportBigdataDistrictFJW(cityCode)
|
|
||||||
//// exportBigdataAliasFJW(cityCode)
|
|
||||||
////
|
|
||||||
//// exportBigdataBuildprojFjw(cityCode)
|
|
||||||
//// exportBigdataCommlayoutpicFjw(cityCode)
|
|
||||||
//// }
|
|
||||||
//// exportEvaluateRate()
|
|
||||||
//
|
|
||||||
////
|
|
||||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
|
||||||
// SSHHelper jschCommand = new SSHHelper();
|
|
||||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
|
||||||
//
|
|
||||||
// exportOk()
|
|
||||||
// }
|
|
||||||
|
|
||||||
def exportBigdataCommunityGJRH(){
|
def exportBigdataCommunityGJRH(){
|
||||||
|
|
||||||
@@ -773,6 +741,145 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
|||||||
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
|
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def exportBigdataCityPrice(queryDate){
|
||||||
|
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||||
|
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
|
||||||
|
if(CommonDatautils.isWindow()){
|
||||||
|
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
|
||||||
|
}
|
||||||
|
def allRows = []
|
||||||
|
for(def city : citys) {
|
||||||
|
def cityCode = cityCodeMap.get(city)
|
||||||
|
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
def data = []
|
||||||
|
data << r.CITYID
|
||||||
|
data << r.YMID
|
||||||
|
data << r.SALEPRICE?r.SALEPRICE:''
|
||||||
|
data << r.RENTPRICE?r.RENTPRICE:''
|
||||||
|
if(r.SALECOUNT>0) {
|
||||||
|
data << r.SALECOUNT
|
||||||
|
}else{
|
||||||
|
data << ''
|
||||||
|
}
|
||||||
|
if(r.RENTCOUNT>0){
|
||||||
|
data << r.RENTCOUNT
|
||||||
|
}else{
|
||||||
|
data << ''
|
||||||
|
}
|
||||||
|
data << r.UPDATETIME
|
||||||
|
data << r.HOUSEUSE
|
||||||
|
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
|
||||||
|
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
|
||||||
|
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
|
||||||
|
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
|
||||||
|
data << r.DATATYPE
|
||||||
|
data << r.SDATERANGE
|
||||||
|
data << r.RDATERANGE
|
||||||
|
data << r.SOURCECODE
|
||||||
|
allRows << data
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||||
|
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||||
|
println "BIGDATA_CITYPRICE 上传Ftp完成......"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def exportBigdataDistPrice(queryDate){
|
||||||
|
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||||
|
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
|
||||||
|
if(CommonDatautils.isWindow()){
|
||||||
|
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
|
||||||
|
}
|
||||||
|
def allRows = []
|
||||||
|
for(def city : citys) {
|
||||||
|
def cityCode = cityCodeMap.get(city)
|
||||||
|
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";//
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
def data = []
|
||||||
|
data << r.CITYID
|
||||||
|
data << r.YMID
|
||||||
|
data << r.DISTID
|
||||||
|
data << r.SALEPRICE?r.SALEPRICE:''
|
||||||
|
data << r.RENTPRICE?r.RENTPRICE:''
|
||||||
|
if(r.SALECOUNT>0) {
|
||||||
|
data << r.SALECOUNT
|
||||||
|
}else{
|
||||||
|
data << ''
|
||||||
|
}
|
||||||
|
if(r.RENTCOUNT>0){
|
||||||
|
data << r.RENTCOUNT
|
||||||
|
}else{
|
||||||
|
data << ''
|
||||||
|
}
|
||||||
|
data << r.UPDATETIME
|
||||||
|
data << r.HOUSEUSE
|
||||||
|
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
|
||||||
|
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
|
||||||
|
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
|
||||||
|
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
|
||||||
|
data << r.DATATYPE
|
||||||
|
data << r.SDATERANGE
|
||||||
|
data << r.RDATERANGE
|
||||||
|
data << r.SOURCECODE
|
||||||
|
allRows << data
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||||
|
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||||
|
println "BIGDATA_DISTPRICE 上传Ftp完成......"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def exportBigdataCommPrice(queryDate){
|
||||||
|
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||||
|
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
|
||||||
|
if(CommonDatautils.isWindow()){
|
||||||
|
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
|
||||||
|
}
|
||||||
|
def allRows = []
|
||||||
|
for(def city : citys) {
|
||||||
|
def cityCode = cityCodeMap.get(city)
|
||||||
|
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
def data = []
|
||||||
|
data << r.CITYID
|
||||||
|
data << r.YMID
|
||||||
|
data << r.COMMID
|
||||||
|
data << r.SALEPRICE?r.SALEPRICE:''
|
||||||
|
if(r.SALECOUNT>0) {
|
||||||
|
data << r.SALECOUNT
|
||||||
|
}else{
|
||||||
|
data << ''
|
||||||
|
}
|
||||||
|
data << r.RENTPRICE?r.RENTPRICE:''
|
||||||
|
if(r.RENTCOUNT>0){
|
||||||
|
data << r.RENTCOUNT
|
||||||
|
}else{
|
||||||
|
data << ''
|
||||||
|
}
|
||||||
|
data << r.UPDATETIME
|
||||||
|
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
|
||||||
|
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
|
||||||
|
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
|
||||||
|
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
|
||||||
|
data << r.HOUSEUSE
|
||||||
|
data << r.SDATERANGE
|
||||||
|
data << r.RDATERANGE
|
||||||
|
data << r.DATATYPE
|
||||||
|
data << r.SOURCECODE
|
||||||
|
allRows << data
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||||
|
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||||
|
println "BIGDATA_COMMPRICE 上传Ftp完成......"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def exportEvaluateRate(){
|
def exportEvaluateRate(){
|
||||||
|
|||||||
+16
@@ -1672,6 +1672,22 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static def queryHouseCalutePrice(city, region, name, startWeekendDate){
|
||||||
|
DBObject queryCondition = new BasicDBObject()
|
||||||
|
queryCondition.put("city", city);
|
||||||
|
queryCondition.put("region", region);
|
||||||
|
queryCondition.put("district_name", name);
|
||||||
|
queryCondition.put('weekend', new BasicDBObject('$gte', startWeekendDate));
|
||||||
|
queryCondition.put("value_type", new BasicDBObject('$in', [1,2]));
|
||||||
|
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(1)
|
||||||
|
def dbo = null;
|
||||||
|
while(cur.hasNext()) {
|
||||||
|
dbo = cur.next()
|
||||||
|
}
|
||||||
|
return dbo;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
|
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
|
||||||
DBObject queryCondition = new BasicDBObject()
|
DBObject queryCondition = new BasicDBObject()
|
||||||
queryCondition.put("city", city);
|
queryCondition.put("city", city);
|
||||||
|
|||||||
+3
-3
@@ -48,13 +48,13 @@ class YouDaDataParser {
|
|||||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||||
|
|
||||||
|
|
||||||
static def statDate = "2022-09-01"
|
static def statDate = "2022-10-01"
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new YouDaDataParser();
|
def task = new YouDaDataParser();
|
||||||
// task.newHouseparse();//友达一手房存入mongo
|
// task.newHouseparse();//友达一手房存入mongo
|
||||||
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
|
// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo
|
||||||
// task.newhouseToSaveEs();//新房保存到es
|
// task.newhouseToSaveEs();//新房保存到es
|
||||||
// task.esfhouseToSaveEs();//二手房保存到es
|
task.esfhouseToSaveEs();//二手房保存到es
|
||||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -398,13 +398,13 @@ public class BeiKeCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles){
|
for(def e : cityEles){
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if(e.text().contains("福州")){
|
// if(e.text().contains("福州")){
|
||||||
def cityMap = [:]
|
def cityMap = [:]
|
||||||
cityMap.city = e.text()
|
cityMap.city = e.text()
|
||||||
cityMap.url = "https:"+href
|
cityMap.url = "https:"+href
|
||||||
list << cityMap
|
list << cityMap
|
||||||
// cityMap.put(e.text(), "https:"+href)
|
// cityMap.put(e.text(), "https:"+href)
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return list
|
return list
|
||||||
|
|||||||
+1
-1
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles) {
|
for(def e : cityEles) {
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
if (!href.contains("fang.lianjia")){
|
||||||
cityMap.put(e.text(), href)
|
cityMap.put(e.text(), href)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
|||||||
|
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def source = "安居客"//
|
def source = "我爱我家"//
|
||||||
def citys = findCrawlCity(source, 202208);
|
def citys = findCrawlCity(source, 202209);
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def count1 = findCount(source, city, 202208);
|
def count1 = findCount(source, city, 202209);
|
||||||
def count2 = findCount(source, city, 202209);
|
def count2 = findCount(source, city, 202210);
|
||||||
println city +" "+count1+" "+count2
|
println city +" "+count1+" "+count2
|
||||||
// println count1
|
// println count1
|
||||||
// println "--- ----"
|
// println "--- ----"
|
||||||
|
|||||||
+3
-2
@@ -24,8 +24,9 @@ class SynchDistrictPrice extends BaseBatch{
|
|||||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||||
|
|
||||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"
|
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
|
||||||
def esSources = [ "搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"]
|
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
|
||||||
|
def esSources = ["贝壳找房":"贝壳找房","链家":"链家","麦田":"麦田"]
|
||||||
|
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
|
|||||||
Reference in New Issue
Block a user