diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 6ef05438e..3b92e612c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -23,7 +23,7 @@ class BigdataBuildingBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_all; def rows = [] def headers = [] for (def city : citys) { @@ -69,7 +69,7 @@ class BigdataBuildingBatch extends BaseBatch{ q.put("city", city) q.put("status", 1) // q.put("name","辽源一村") - def dbcursor = LOU_DONG_STAT.find(q) + def dbcursor = LOU_DONG_STAT.find(q).batchSize(5000) def map = [:] def set = new HashSet() for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index b54b97e70..425f25225 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -27,7 +27,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -35,7 +35,8 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["临夏"]//CcbCommonUtils.citys_all; + def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -43,18 +44,19 @@ class BigdataCommunityBatch extends BaseBatch{ //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + println "starting 。。。。。。。。。" for(def city : citys){ + index ++ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) - println city +" 小区量:"+dists.size() def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) - println city +" "+rets.size() - + println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } @@ -220,7 +222,7 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "" ret << 0 }catch(Exception e){ - println dist.city + " "+dist.name + println dist.city + " "+dist.name+" error" e.printStackTrace() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index e92c8b656..5251916f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -57,8 +57,8 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) +// def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys; -// def ccbCitys = CcbCommonUtils.citys; for(def city : citys){ // if(!ccbCitys.contains(city)){ @@ -327,7 +327,7 @@ class BigdataDealBatch extends BaseBatch{ } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() - ret << trade.city + ret << CcbCommonUtils.getCcbCity(trade.city) if(data){ ret << data.district_code ret << data.district diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index c3affa2bd..e5272b8bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -10,6 +10,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql + /** * 建行模型估值项目 * 区域数据同步 @@ -17,33 +19,74 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class BigdataDistrictBatch extends BaseBatch{ + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { runBatch(new BigdataDistrictBatch(), args); } @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; - def rows = [] - for(def city : citys){ - def datas = findDistrictCodeValue(city) - if(datas){ - rows.addAll(datas) - } - println city+" "+ datas.size() - } - - def allDatas = createDatas(rows) - - def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] - + def datas = readText("D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.txt") def rets = [] - rets << headers - rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); + def index = 0 + for(def line : datas){ + def arrs = line[0].split("\\|@\\|") + def sql = "select count(*) as count from BIGDATA_DISTRICT_GJ where distid='${arrs[0]}'"; + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + if(count==0){ + rets << line + } + } + writeTxt(rets, "D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ_add.txt") +// def citys = CcbCommonUtils.citys_all; +// def rows = [] +// for(def city : citys){ +// def datas = findDistrictCodeValue(city) +// if(datas){ +// rows.addAll(datas) +// } +// println city+" "+ datas.size() +// } +// +// def allDatas = createDatas(rows) +// +// def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] +// +// def rets = [] +// rets << headers +// rets.addAll(allDatas) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") +// FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); } + + def writeTxt(def rows, def path){ + StringBuilder builder = new StringBuilder() + for(def row : rows){ + builder.append(row) + builder.append('\n') + } + org.apache.commons.io.FileUtils.writeStringToFile(new File(path), builder.toString(), "GBK") + } + private def readText(def fileName){ + File file = new File(fileName) + def ret = []; + //对文件内容的每一行进行分割处理,比较常用在处理csv文件 + file.splitEachLine(" ") { + def temp = it + ret << temp; + } + //转为Writable对象,可重定向输出 + file.asWritable() + return ret; + } + + def createDatas(rows){ def allRows = [] def index = 1 @@ -72,16 +115,16 @@ class BigdataDistrictBatch extends BaseBatch{ def findDistrictCodeValue(city){ def q = new BasicDBObject() - q.put("fangjia_city", city) - // q.put("district", new BasicDBObject('$ne',"市辖区")) - def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + q.put("fjw_city", city) +// def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q) + def dbcursor = CcbCommonUtils.FANGJIA_AREA_CODE.find(q) def lists = [] for(def db : dbcursor){ if(db.district_code){ def m = [:] m.id = db._id.toString() m.dist_name = db.district - m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "") + m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fjw_district, "") m.city_id = db.city_code m.dist_code = db.district_code lists << m diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 6e1b76aa5..8cf460e6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -538,7 +538,7 @@ class BigdataForsaleBatch extends BaseBatch{ } rowMap.houseId = seawater.id - rowMap.city = city + rowMap.city = CcbCommonUtils.getCcbCity(city) //rowMap.region =seawater.region //rowMap.address = seawater.address diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index e4f47e2fd..3ef4cee91 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,12 +16,16 @@ import groovy.sql.Sql class CcbCommonUtils { // + public static def citys_all = [ "长春","上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] public static def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] - public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com") +// public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com") + + public static final DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com") + public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com") @@ -34,39 +38,58 @@ class CcbCommonUtils { public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","巴中":"巴州","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"] + public static DSCollection dataStoreConnection = null; public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + + public static def findCityCodeValue(){ - def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] + def cityCodeMap = [:] + FANGJIA_AREA_CODE.find().each {it-> + def code = cityCodeMap.get(it.fjw_city) + if(!code){ + cityCodeMap.put(it.fjw_city, it.city_code.toString()) + } + } +// def oldCityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"] return cityCodeMap; } + public static def getCcbCity(def city){ + if(city && specialCityMap.containsKey(city)){ + return specialCityMap.get(city) + } + return city; + } + public static def findAllDistrictCodeValue(city){ def q = new BasicDBObject() - q.put("fangjia_city", city) + q.put("fjw_city", city) //q.put("district", new BasicDBObject('$ne',"市辖区")) - def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1)) def map = [:] for(def db : dbcursor){ - map.put(db.fangjia_district, db) + map.put(db.fjw_district, db) } return map; } public static def findAllCityDistrictCodeValue(city){ def q = new BasicDBObject() - def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1)) + def dbcursor = FANGJIA_AREA_CODE.find(q).sort(new BasicDBObject("_id", 1)) def map = [:] for(def db : dbcursor){ - def distMap = map.get(db.fangjia_city) + def distMap = map.get(db.fjw_city) if(!distMap){ distMap = [:] - map.put(db.fangjia_city, distMap) + map.put(db.fjw_city, distMap) } - distMap.put(db.fangjia_district, db) + distMap.put(db.fjw_district, db) } return map; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 74474aff6..6b972553e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpUtils { - static def ftpServer = "192.168.0.45" + static def ftpServer = "192.168.0.45"//192.168.0.45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 6dcffbacf..e01954de1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -370,13 +370,13 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = ["上海","广州","东莞"]//CcbCommonUtils.citys + def citys = CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def month = 202207 + def month = 202208 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202208) + def rets = findPeGteThirdParty(city, 202209) def total = 0 println city +" "+rets.size() for(def m : rets){ @@ -388,7 +388,7 @@ class DealPriceDataProcess { } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30" def rets = [] sqlInstance236.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index 94741174c..ed5ed5f03 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -62,7 +62,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ ret << dist._id.toString() ret << dist.name ret << cityCodeDatas.get(dist.city) - ret << dist.city + ret << CcbCommonUtils.getCcbCity(dist.city) ret << areaData.district_code ret << areaData.district ret << ps.avg_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e229de8e5..0fbc2daa8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -15,6 +15,8 @@ class ExportOracleData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + static main(args) { def task = new ExportOracleData() task.run() @@ -27,18 +29,18 @@ class ExportOracleData { def new_dists = [] //不会变 exportEvaluationRateData() - exportBigdataPoiData() - exportBigdataDistrictData() + exportBigdataPoiData()// + exportBigdataDistrictData()// - exportGuidedPrice()//随人工改变 - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// // //处理小区情况 - exportBigdataCommunity() - exportCommunityStaging() - exportCommunityBuileType() + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// - exportThirdPartyPrice2(lastMonth, new_dists) + exportThirdPartyPrice2(lastMonth, new_dists)// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 @@ -129,24 +131,29 @@ class ExportOracleData { def exportBigdataPoiData() { - def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj"; + def citys = CcbCommonUtils.citys_all def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.CITYID - data << r.DISTID - data << r.ID - data << r.POI_NAME - data << r.TYPE - data << r.TYPE_CODE - data << r.LATITUDE - data << r.LONGITUDE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; @@ -163,8 +170,8 @@ class ExportOracleData { } def exportCommunityStaging() { + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; - def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID @@ -193,9 +200,10 @@ class ExportOracleData { // def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def allRows = [] -// for(def city : citys){ -// def cityid = cityCodeDataMap.get(city) - def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj"; + def citys = CcbCommonUtils.citys_all + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; sqlInstance.eachRow(sql, { r -> def data = [] data << r.CITYID @@ -208,7 +216,8 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); -// } + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" @@ -224,28 +233,32 @@ class ExportOracleData { } def exportBigdataDistrictData() { - - def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj"; + def citys = CcbCommonUtils.citys_all def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.DISTID - data << r.DISTNAME - data << r.DISTSPELL - data << r.CITYID - data << r.DISTCODE - data << r.SEQ - data << r.DISTTYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.LJXZQH - data << r.LJXZQHDM - data << r.XZQHDM - data << r.WORKDATE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -262,26 +275,31 @@ class ExportOracleData { def exportGuidedPrice() { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj"; + def citys = CcbCommonUtils.citys_all def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMID - data << r.COMMNAME - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.DEALDATE - data << r.AVG_PRICE - data << r.GUIDING_PERSON - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -296,25 +314,30 @@ class ExportOracleData { } def exportCommunityGuidedPrice(def date) { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')"; + def citys = CcbCommonUtils.citys_all def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMID - data << r.COMMNAME - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.AVGPRICE - data << r.SOURCETYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -331,30 +354,34 @@ class ExportOracleData { def exportBigdataDeal(def dealDate, def new_dists) { def allRows = [] - if(new_dists){ - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" - def rets1 = dealData(sql1) + def citys = CcbCommonUtils.citys_all + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" + def rets1 = dealData(sql1) - def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where distid in ${newstr}" - def rets2 = dealData(sql2) - allRows.addAll(rets1) - allRows.addAll(rets2) - }else{ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; - allRows = dealData(sql) + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows = dealData(sql) + } } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; @@ -363,8 +390,6 @@ class ExportOracleData { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" } def headers = [] -//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_DEAL_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) @@ -497,25 +522,6 @@ class ExportOracleData { def exportThirdPartyPrice2(price_date, new_dists) { - def allRows = [] - if(new_dists) { - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + - "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + - "price_date=${price_date} and distid not in ${newstr} "; - def rets1 = getThirdPartyDatas(sql1) - allRows.addAll(rets1) - - def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + - "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + - "distid in ${newstr} "; - def rets2 = getThirdPartyDatas(sql2) - allRows.addAll(rets2) - }else{ - def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} "; - allRows = getThirdPartyDatas(sql) - } - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -523,9 +529,38 @@ class ExportOracleData { // fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = [] -//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + def index = 0 + def citys = CcbCommonUtils.citys_all + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) } @@ -552,45 +587,51 @@ class ExportOracleData { } def exportBigdataCommunity(){ - def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + - "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100) + + def citys = CcbCommonUtils.citys_all def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.DISTID - data << r.COMMID - data << r.DISTNAME - data << r.COMMNAME - data << r.ADDRESS - data << r.LATITUDE - data << r.LONGITUDE - data << r.LANDAREA - data << r.BUILDAREA - data << r.GREENINGRATE - data << r.PLOTRATIO - data << r.PARKINGCNT - data << r.BUILDYEAR - data << r.HOUSEUSE - data << r.ZTS - data << r.WORKDATE - data << r.UPDATEFLAG - data << r.UPDATETIME - data << r.COMMTYPE - data << r.COMMALIAS - data << r.STREET - data << r.STREETNO - data << r.IFNEW - data << "" - data << r.REMARK - data << r.CREATETIME - data << r.SOURCECODE - data << r.XZJDB - data << r.LJXZQHDM - data << r.LJXZQH - data << r.STATUS - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -607,8 +648,7 @@ class ExportOracleData { def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] - def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def citys = CcbCommonUtils.citys_all; def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" @@ -616,9 +656,9 @@ class ExportOracleData { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 for(def city : citys){ - def cityCode = cityCodeDataMap.get(city) - + def cityCode = cityCodeMap.get(city) def allRows = [] if(new_dists){ @@ -636,7 +676,13 @@ class ExportOracleData { allRows = getHouseDatas(sql) } if(allRows.size()>0){ - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 0715fc907..b28516075 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -27,9 +27,46 @@ class FjwPricePkCcbPrice { def task = new FjwPricePkCcbPrice() // task.run() // task.updatePrice() - task.addFjwPrice() +// task.addFjwPrice() +// task.addCcbPrice() + task.addMarketPrice() } + def addMarketPrice(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\拍卖南京.xlsx",{d-> + if(d['D']) { + d['K'] = d['D'] + }else if(d['H']){ + d['K'] = d['H'] + } + + }) + } + + def addCcbPrice(){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\新老算法价格差异大小区.xlsx",{d-> + if(d['B']){ + def cityId = cityCodeDataMap.get(d['B']) + def dist = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district") + if(dist){ + def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209) + if(map){ + d['I'] = map.avg_price + d['J'] = map.calc_type + d['k'] = map.credit + } + } + +// def dist = CommonDatautils.findDistrictById(d['B']) +// if(dist){ +// d['J'] = dist.status +// } + } + }) + } + + def updatePrice(){ def index = 0 ExcelUtils.multiThreadingModify("D:\\数据\\建行\\测试\\ccb.xlsx",{d-> @@ -174,6 +211,27 @@ class FjwPricePkCcbPrice { } + def getCommunityBasePriceGjByCommid(cityid, commid, yearMonth) { + def sql = "select AVG_PRICE, CALC_TYPE,CONFIDENCE from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} and commid='${commid}'"; + def map = [:] + sqlInstance236.eachRow(sql, { r -> + map.avg_price = r.AVG_PRICE + // 计算类型,1:人工估值 2:样本计算 3:神经网络 + if(r.CALC_TYPE){ + if(r.CALC_TYPE == 1){ + map.calc_type = "人工估值" + }else if(r.CALC_TYPE == 2){ + map.calc_type = "样本计算" + }else if(r.CALC_TYPE == 3){ + map.calc_type = "神经网络" + } + } + map.credit = r.CONFIDENCE + }); + return map + } + + def batchCreateFile(ret, fileName, headers, charset){ OutputStream out = new FileOutputStream(fileName); def sb = new StringBuffer(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 666909b2e..936022f98 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-08-24", "2022-09-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-09-22", "2022-09-23") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) @@ -147,7 +147,7 @@ class GuiDingPriceBatch extends BaseBatch{ ret << dist._id.toString() ret << dist.name ret << cityCodeDatas.get(dist.city) - ret << dist.city + ret << CcbCommonUtils.getCcbCity(dist.city) ret << areaData.district_code ret << areaData.district ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 44494cad2..31ae30039 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -977,25 +977,50 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\moban_ccb_all.xlsx" +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx" +//def datas = ExcelUtils.read(fileName, 0, 1) +//def keys = [] +//for(def d : datas){ +// def key = d['A']+"##"+d['B']+"##"+d['C'] +// keys << key +//} +// +//def fileName1 = "D:\\数据\\建行\\房产估值\\指导价\\moban_ccb_0922.xlsx" +//def datas1 = ExcelUtils.read(fileName1, 0, 1) +//def keys1 = [] +//for(def d : datas1){ +// def key = d['A']+"##"+d['B']+"##"+d['C'] +// keys1 << key +//} +// +//for(def k : keys1){ +// if(!keys.contains(k)){ +// println k +// } +//} + + + +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx" //ExcelUtils.multiThreadingModify(fileName, {d-> // if(d['A']){ -//// def region = RegionData.formatRegion(d['B'], d['C']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") -//// if(dist){ -//// d['E'] = dist._id.toString() -//// } -//// } -//// if(!d['Q']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -//// +// def region = RegionData.formatRegion(d['A'], d['B']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") // if(dist){ -// d['H'] = dist.region -//// d['G'] = dist.name -// d['I'] = dist.status +// d['E'] = dist.region +// d['F'] = dist._id.toString() // } +// } +//// if(!d['Q']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +////// +//// if(dist){ +//// d['H'] = dist.region +////// d['G'] = dist.name +//// d['I'] = dist.status +//// } //// } // // } @@ -1003,13 +1028,13 @@ def fileName = "D:\\ -ExcelUtils.modify(fileName, {d-> - if(d['A']){ - def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" - println sql - sqlInstance236.executeUpdate(sql) - } -},0) +//ExcelUtils.modify(fileName, {d-> +// if(d['A']){ +// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" +// println sql +// sqlInstance236.executeUpdate(sql) +// } +//},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbRegionCode.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbRegionCode.groovy new file mode 100644 index 000000000..b2541547c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbRegionCode.groovy @@ -0,0 +1,26 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + + +DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com") + +def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\建行最终区域编码.xlsx", 0, 1) +for(def d : datas){ + def dbo = new BasicDBObject() + dbo.put("province",d['C']) + dbo.put("city", d['D']) + dbo.put("district", d['G']) + dbo.put("city_code", d['E'].toInteger()) + dbo.put("district_code",d['F'].toInteger()) + dbo.put("fjw_city", d['A']) + dbo.put("fjw_district", d['B']) + dbo.put("c_date", new Date()) + FANGJIA_AREA_CODE.save(dbo) +} + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index be808deaa..b8116db22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_all; def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { @@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch { // }) // } -// addBigdataDistrictFJW() + addBigdataDistrictFJW() - def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); - - def index = 0 - for (def city : citys) { - index++ - def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); // -// for(int i=0;i<10;i++){ -// if(districts.size()==0){ -// Thread.sleep(1000*60) -// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) -// }else{ -// break +// def index = 0 +// for (def city : citys) { +// index++ +// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +//// +//// for(int i=0;i<10;i++){ +//// if(districts.size()==0){ +//// Thread.sleep(1000*60) +//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +//// }else{ +//// break +//// } +//// } +// +// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) +// +// for(def dist : districts){ +// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) +// if(obj){ +// if(obj.alias && obj.alias.size()>0){ +// dist.commalias = obj.alias.get(0) +// }else{ +// dist.commalias = "" +// } +// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) +// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) +// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); +// dist.houseuse = "11" +// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) +// dist.compid = new ObjectId() //物业公司id +// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) // } // } - - def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) - - for(def dist : districts){ - def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) - if(obj){ - if(obj.alias && obj.alias.size()>0){ - dist.commalias = obj.alias.get(0) - }else{ - dist.commalias = "" - } - dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) - dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) - dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); - dist.houseuse = "11" - dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) - dist.compid = new ObjectId() //物业公司id - dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) - } - } - println index +" "+city +" 小区加载完成 "+districts.size() - addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 1.小区表完成" - addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 2.小区建筑结构表" - addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 3.小区房屋类型表" - addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 4.小区建筑年代表" - addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 5.小区费用表" - addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 6.小区房屋用途表" - addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) - println index +" "+ city + " 7.地理位置表" - -// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 - - } +// println index +" "+city +" 小区加载完成 "+districts.size() +// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 1.小区表完成" +// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 2.小区建筑结构表" +// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 3.小区房屋类型表" +// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 4.小区建筑年代表" +// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 5.小区费用表" +// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 6.小区房屋用途表" +// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) +// println index +" "+ city + " 7.地理位置表" +// +//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 +// +// } } def addBigdataCommunityGJRH(def districts, def cityid){ @@ -557,14 +557,21 @@ class BigdataCommunityGJRHBatch extends BaseBatch { def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){ def rets = [] + def num = 0 for(def d : districts) { + num = num + 1 def ret = [] + def detailMap = citySeqnoMap.get(d.cityid) def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap); ret << d.cityid ret << "5" ret << d.commid ret << "1" - ret << seqno + if(detailMap && seqno){ + ret << seqno + }else{ + ret << num + } ret << d.lng ret << d.lat ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index e50059409..3907b73d5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -22,25 +22,28 @@ class ExportDistrictRongHe extends BaseBatch{ runBatch(new ExportDistrictRongHe(), args); } + def citys = CcbCommonUtils.citys + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + @Override void process(Object args) { def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] - //按天导(每天一个文件夹) + //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛� exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH() - exportBigdataBuildingFJW() - exportBigdataCommfeeGj() + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// - exportBigdataCommbuildstruGJ() - exportBigdataCommhousetypeGJ() - exportBigdataCommbuildyearGj() - exportBigdataCommhouseuseGJ() - exportBigdataPositionGJ() - exportBigdataDistrictFJW() - exportBigdataAliasFJW() + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// exportBigdataBuildprojFjw() exportBigdataCommlayoutpicFjw() @@ -59,14 +62,14 @@ class ExportDistrictRongHe extends BaseBatch{ // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // // -//// //检查ok文件是否上传 +//// //妫�鏌k鏂囦欢鏄惁涓婁紶 //// exportAllBigdataDealFJW() //// for(def city : citys) { //// def cityCode = cityCodeDataMap.get(city) //// -// //按天导(一个文件夹下) +// //鎸夊ぉ瀵�(涓�涓枃浠跺す涓�) // //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// //按天导(每天一个文件夹) +// //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛� // exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) // //// exportBigdataCommunityGJRH(cityCode) @@ -94,261 +97,308 @@ class ExportDistrictRongHe extends BaseBatch{ // exportOk() // } - def exportBigdataCommunityGJRH(cityCode){ - def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}' - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.COMMNAME - data << r.COMMALIAS - data << r.DISTID - data << r.STREET - data << r.STREETNO - data << r.ADDRESS - data << r.LANDAREA - data << r.BUILDAREA - data << r.COMMTYPE - data << r.IFNEW - data << r.GREENINGRATE - data << r.PLOTRATIO - data << r.PARKINGCNT - data << '' - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + def exportBigdataCommunityGJRH(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" + def index = 0 + for(def city : citys){ + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMMNAME + data << r.COMMALIAS + data << r.DISTID + data << r.STREET + data << r.STREETNO + data << r.ADDRESS + data << r.LANDAREA + data << r.BUILDAREA + data << r.COMMTYPE + data << r.IFNEW + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << '' + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + + } + println "BIGDATA_COMMUNITY_FJW 鏁版嵁鏂囦欢杈撳嚭瀹屾垚......" CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" + println "BIGDATA_COMMUNITY_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommbuildstruGJ(){ - def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDSTRU - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDSTRU + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......" + println "BIGDATA_COMMBUILDSTRU_FJW 涓婁紶Ftp瀹屾垚......" } - def exportBigdataCommhousetypeGJ(cityCode){ - def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}' + def exportBigdataCommhousetypeGJ(){ + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.HOUSETYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......" + println "BIGDATA_COMMHOUSETYPE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommbuildyearGj(){ - def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDYEAR - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDYEAR + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......" + println "BIGDATA_COMMBUILDYEAR_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommfeeGj(){ - def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}' + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.COMPID - data << r.FEE - data << r.HOUSEUSE - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.FEETYPE - data << r.NOTE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMPID + data << r.FEE + data << r.HOUSEUSE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.FEETYPE + data << r.NOTE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMFEE_FJW 上传Ftp完成......" + println "BIGDATA_COMMFEE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommhouseuseGJ(){ - def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.HOUSEUSE - data << r.HOUSECNT - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSEUSE + data << r.HOUSECNT + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......" + println "BIGDATA_COMMHOUSEUSE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataPositionGJ(){ - def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.OBJECTTYPE - data << r.OBJECTID - data << r.POSITIONTYPE - data << r.SEQNO - data << r.LONGITUDE - data << r.LATITUDE - data << r.ALTITUDE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.OBJECTTYPE + data << r.OBJECTID + data << r.POSITIONTYPE + data << r.SEQNO + data << r.LONGITUDE + data << r.LATITUDE + data << r.ALTITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_POSITION_FJW 上传Ftp完成......" + println "BIGDATA_POSITION_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataDistrictFJW(){ - def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.DISTID - data << r.DISTNAME - data << r.DISTSPELL - data << r.CITYID - data << r.DISTCODE - data << r.SEQ - data << r.DISTTYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" + println "BIGDATA_DISTRICT_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataAliasFJW(){ - def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.city_id - data << r.ca_id - data << r.address_id - data << r.address_name - data << r.status - data << r.exp_dt - data << r.version - data << r.is_key_name - data << r.data_source - data << r.description - data << r.alias_grade - data << r.addr_type_cd - data << r.ifexamine - data << r.ifdisplay - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW where city_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.city_id + data << r.ca_id + data << r.address_id + data << r.address_name + data << r.status + data << r.exp_dt + data << r.version + data << r.is_key_name + data << r.data_source + data << r.description + data << r.alias_grade + data << r.addr_type_cd + data << r.ifexamine + data << r.ifdisplay + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "ADDRESS_ALIAS_FJW 上传Ftp完成......" + println "ADDRESS_ALIAS_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataDealFJW(dealdate){ @@ -368,7 +418,7 @@ class ExportDistrictRongHe extends BaseBatch{ def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } def allRows = [] @@ -453,7 +503,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" + println "BIGDATA_DEAL_FJW 涓婁紶Ftp瀹屾垚......${curStartStr}" def curEndStr = DateUtils.add(curStartStr, 1) curStartDateStr = curEndStr } @@ -468,31 +518,36 @@ class ExportDistrictRongHe extends BaseBatch{ def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } def allRows = [] - if(curDate.toString().endsWith("28")){ - if(new_dists){ - def newstr = "('"+new_dists.join("','")+"')" - def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + - "where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" + - " distid not in ${newstr} ";//CITYID='${cityCode}' and - def rets1 = createDeal(sql1) - def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + - " where distid in ${newstr}" - def rets2 = createDeal(sql2) - allRows.addAll(rets1) - allRows.addAll(rets2) - }else{ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and - allRows = createDeal(sql) + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(curDate.toString().endsWith("28")){ + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + + "where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" + + " distid not in ${newstr} ";//CITYID='${cityCode}' and + def rets1 = createDeal(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + + " where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = createDeal(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";// + allRows = createDeal(sql) + } } } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + println "BIGDATA_DEAL_FJW 涓婁紶Ftp瀹屾垚......${endDateStr}" } def createDeal(sql){ @@ -653,54 +708,68 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "BIGDATA_DEAL_FJW 导出完成......" + println "BIGDATA_DEAL_FJW 瀵煎嚭瀹屾垚......" } - def exportBigdataBuildingFJW(cityCode){ - def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}' - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.BUILDALIAS - data << r.DISTID - data << r.COMMID - data << r.BUILDYEAR - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.BUILDSTRU - data << r.FLOORCNT - data << r.UNITCNT - data << r.HOUSECNT - data << r.IFELEVATOR - data << r.ELEVATORCNT - data << r.ELEVATORHOUSE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.VALID_FLAG - allRows << data - }); + def exportBigdataBuildingFJW(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + def index = 0 + for(def city : citys) { + + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.BUILDALIAS + data << r.DISTID + data << r.COMMID + data << r.BUILDYEAR + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.FLOORCNT + data << r.UNITCNT + data << r.HOUSECNT + data << r.IFELEVATOR + data << r.ELEVATORCNT + data << r.ELEVATORHOUSE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.VALID_FLAG + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_BUILDING_FJW 上传Ftp完成......" + println "BIGDATA_BUILDING_FJW 涓婁紶Ftp瀹屾垚......" } @@ -724,40 +793,40 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "EVALUATE_RATE_FJW 上传Ftp完成......" + println "EVALUATE_RATE_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" + println "BIGDATA_BUILDPROJ_FJW 涓婁紶Ftp瀹屾垚......" } def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" + println "BIGDATA_COMMLAYOUTPIC_FJW 涓婁紶Ftp瀹屾垚......" } def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" if(CommonDatautils.isWindow()){ - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + dataCodeFileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" } FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",dataCodeFileName, curDate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionUpdate.groovy index 51a06b0c3..deff62712 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionUpdate.groovy @@ -18,9 +18,10 @@ class RegionUpdate { } def run1(){ - def districts = CommonDatautils.findAllFiledDistrict("district","漳州"); + def city = "四平" + def districts = CommonDatautils.findAllFiledDistrict("district",city); - def regions = CommonDatautils.getRegionListByCity("漳州") + def regions = CommonDatautils.getRegionListByCity(city) def index = 0 def addIndex = 0 DistrictRegionTransfer transfer = new DistrictRegionTransfer() @@ -28,6 +29,9 @@ class RegionUpdate { if(districts){ //根据经纬度获取经纬度所在区域 for(def dist : districts){ + if(dist.region!="公主岭"){ + continue + } index++ if(!regions.contains(dist.region)){ @@ -35,28 +39,28 @@ class RegionUpdate { println regionErrorIndex +" "+dist.name } -// def region = getRegionByLocation(dist.lng/10000000000d+","+dist.lat/10000000000d); -// if(!region){ -// def detailMap = getRegionByAddress("漳州", dist.address); -// if(detailMap && detailMap.region && detailMap.region != dist.region){ -// addIndex++ -// dist.lng = (detailMap.lng.toDouble()*10000000000).longValue() -// dist.lat = (detailMap.lat.toDouble()*10000000000).longValue() -// -// def gmap = GeocodeUtils.getGaodeToBaidu(detailMap.lng.toDouble(), detailMap.lat.toDouble()) -// dist.lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue() -// dist.lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue() -// -// println "[${addIndex}/${index}]"+dist.region +" "+detailMap.region+" "+ dist.name -// transfer.transfer(dist, dist.city, detailMap.region) -// } -// }else{ -// if(region && region!=dist.region){ -// addIndex++ -// println "[${addIndex}/${index}]"+region +" "+dist.region+" "+ dist.name -// transfer.transfer(dist, dist.city, region) -// } -// } + def region = getRegionByLocation(city, dist.lng/10000000000d+","+dist.lat/10000000000d); + if(!region){ + def detailMap = getRegionByAddress(city, dist.address); + if(detailMap && detailMap.region && detailMap.region != dist.region){ + addIndex++ + dist.lng = (detailMap.lng.toDouble()*10000000000).longValue() + dist.lat = (detailMap.lat.toDouble()*10000000000).longValue() + + def gmap = GeocodeUtils.getGaodeToBaidu(detailMap.lng.toDouble(), detailMap.lat.toDouble()) + dist.lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue() + dist.lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue() + + println "[${addIndex}/${index}]"+dist.region +" "+detailMap.region+" "+ dist.name + transfer.transfer(dist, dist.city, detailMap.region) + } + }else{ + if(region && region!=dist.region){ + addIndex++ + println "[${addIndex}/${index}]"+region +" "+dist.region+" "+ dist.name + transfer.transfer(dist, dist.city, region) + } + } } } } @@ -85,14 +89,16 @@ class RegionUpdate { return detail } - def getRegionByLocation(def location){ + def getRegionByLocation(city, def location){ def json = GaoDeApiUtils.geocode(location); - def city = "漳州" def region = null; if (json && json.info=="OK") { def regeocode = json.regeocode if(regeocode && regeocode.addressComponent){ region = regeocode.addressComponent.district + if(!region){ + region = regeocode.addressComponent.township + } } } if(region){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy index 2e9a536ef..fe2ab3332 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy @@ -11,9 +11,10 @@ import org.apache.lucene.queryparser.classic.QueryParser class DongGuanBankTest { static main(args) { def task = new DongGuanBankTest() - def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx" + def path = "D:\\数据\\东莞银行\\东莞银行测试_输出单价(2).xlsx" // task.run(path) - task.addPrice(path) +// task.addPrice(path) + task.addMinMaxPrice(path) } def run(path){ @@ -92,7 +93,7 @@ class DongGuanBankTest { def set = null try{ def aliasStr = escapeExprSpecialWord(alias.join("','")) - String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')" + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210901 and region='${region}' and district_name in ('${aliasStr}') and root='anjuke.com'" set = new HashSet() ElasticUtils.executeSqlQuery(sql,{data-> @@ -132,4 +133,26 @@ class DongGuanBankTest { return avg_price } + def addMinMaxPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['V']){ + def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district") + if(dist){ + def alias = [] + if(!dist.alias){ + alias = [] + }else{ + alias = dist.alias + } + alias << dist.name + def map = findMaxAvgPrice(dist.city, dist.region, alias) + if(map){ + d['V'] = map.max + d['W'] = map.min + } + } + } + }) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index be10cf533..3d79e39fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -19,9 +19,44 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() - task.houseRun() +// task.houseRun() // task.statHouseUpdateCount() + + task.statDistHouse() + } + + def statDistHouse(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\大连有效小区确认_2256--查好-1.xlsx",{d-> + if(d['B']){ + def alias = [] + if(d['G']){ + def s = d['G'].replace("[", "").replace("]", "").replace("'","").replace('"',"") + def arrs = null + if(s){ + arrs = s.split(",") + } + if(arrs){ + arrs = arrs as List + if(!arrs.contains(d['D'])){ + arrs << d['D'] + } + alias = arrs + }else{ + alias << d['D'] + } + }else{ + alias << d['D'] + } + + def distCount = findHouseCount(d['B'], d['C'], alias, 20190101, 20221001) + if(distCount){ + d['E'] = distCount + } + println d['B']+" "+d['C']+" "+d['D']+" "+distCount + } + + }) } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy new file mode 100644 index 000000000..444fddbfc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/JnBlankLog.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.util.ElasticUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +EsDataSource dataSource = new EsDataSource(); +String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; + +findLogs() + +def findLogs(){ + def total = 0 + try{ + +// String sql = "select * from fangjia-log* where request like '/jnBank%'" +// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery(); + def q = QueryBuilders.fuzzyQuery("request", "/jnBank") + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q) + def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder); + println houses.size() + def set = new HashSet() + if(houses) { + for (def d : houses) { + def request = d.request + if(request){ + def arrs = request.split("&") + if(arrs){ + for(def a : arrs){ + if(a.contains("CITY_CD")){ + def list = a.split("=") + if(list && list.size()>=2){ + set.add(list[1]) + } + } + } + } + } + } + } + + println set + + +// ElasticUtils.executeSqlQuery(sql,{ data-> +// println data.request +// },"192.168.0.25") + }catch(Exception e){ + e.printStackTrace() + } + + return total +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 9bdcef9c4..e61f3588a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.rabbitmq.client.AMQP import com.rabbitmq.client.impl.AMQImpl @@ -35,11 +36,12 @@ class PaimaiDataSend { static main(args) { def task = new PaimaiDataSend() // task.jdRunByMongo() -// task.jdRun() -//// task.aliZcRun() + +// task.aliZcRun() // task.aliZcRun2() // task.aliFpRun() // task.rmfysszcRun() +// task.jdRun() // task.checkUrl() task.paimaiSendToEs() @@ -67,7 +69,7 @@ class PaimaiDataSend { def index = 0 def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"])) - q.put("province","广东省") + q.put("province","江苏省") // q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml") def dbcursor = rmfysszc_coll.find(q).batchSize(3000) for(def d : dbcursor){ @@ -151,7 +153,7 @@ class PaimaiDataSend { def index = 0 def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"])) - q.put("province","广东") + q.put("province","江苏") // q.put("parser_status", new BasicDBObject('$ne',3)) // q.put("url","https://paimai.jd.com/115669796") def dbcursor = jd_coll.find(q).batchSize(3000) @@ -223,11 +225,12 @@ class PaimaiDataSend { def aliZcRun(){ def pm_q = new BasicDBObject() - pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"])) -// pm_q.put("city","苏州") +// pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"])) + pm_q.put("province","江苏") // pm_q.put("region","姑苏") // pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm") - pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15"))) +// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15"))) + pm_q.put("mDate", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2022-10-01"))) Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE); pm_q.put("url", new BasicDBObject('$regex', pattern)) // pm_q.put("mappingEstateType", null) @@ -235,8 +238,8 @@ class PaimaiDataSend { def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000) println dbcursor.count() for(def d : dbcursor){ - def flag = sendAliZc(d.url) -// def flag = sendAli(d.url) +// def flag = sendAliZc(d.url) + def flag = sendAli(d.url) if(!flag){ println "不存在:"+d.url } @@ -331,14 +334,14 @@ class PaimaiDataSend { def index = 0 def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) - q.put("province","广东") + q.put("province","江苏") // q.put("parser_status", new BasicDBObject('$ne',2)) -// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm") + q.put("url","https://sf-item.taobao.com/sf_item/665795393092.htm") //https://sf-item.taobao.com/sf_item/657131607486.htm // q.put("parser_status", new BasicDBObject('$exists',false)) // q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01"))) // q.put("area", new BasicDBObject('$lt', 30)) - def dbcursor = ali_fp_coll.find(q).batchSize(5000) + def dbcursor = ali_fp_coll.find(q).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def rets = [] @@ -351,8 +354,12 @@ class PaimaiDataSend { JSON result = new JSONObject() result.put("content", d.content) - def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) - result.put("trades", bid_content) + try{ + def bid_content = net.sf.json.JSONObject.fromObject(d.bid_content) + result.put("trades", bid_content) + }catch(Exception e){ + println "trades结果格式问题 "+d.url + } result.put("province", d.province) result.put("city", d.city) result.put("region", d.region) @@ -365,7 +372,7 @@ class PaimaiDataSend { json.put("result", result) rets << json - if(rets.size()==10000){ + if(rets.size()==5000){ println "共:"+rets.size() send(rets) rets.clear() @@ -391,7 +398,7 @@ class PaimaiDataSend { CommonUtils.sendMessage("result2parse2dev", packedMsg) def udata = new BasicDBObject() - udata.put("parser_status", 2) + udata.put("parser_status", 0) udata.put("m_date", new Date()) ali_fp_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata)) println "send: "+totalCount.get() @@ -411,11 +418,12 @@ class PaimaiDataSend { def index = 0 def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"])) - q.put("province","广东") +// q.put("city", "上海") + q.put("province","江苏") // q.put("parser_status", new BasicDBObject('$ne',1)) // q.put("region","姑苏") -// q.put("url","https://susong-item.taobao.com/auction/652481594083.htm") - def dbcursor = ali_zc_coll.find(q).batchSize(5000) + q.put("url","https://susong-item.taobao.com/auction/674736290116.htm") + def dbcursor = ali_zc_coll.find(q).batchSize(5000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def rets = [] for(def d : dbcursor){ index++ @@ -514,7 +522,8 @@ class PaimaiDataSend { def getPaimaiDatas(){ def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) - q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-15"))) + q.put("city","苏州") + q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-10-09"))) def dbcursor = paimai43_COLL.find(q).batchSize(5000) def rets = [] for(def d : dbcursor){ @@ -545,6 +554,7 @@ class PaimaiDataSend { o.matchCity = d.matchCity o.matchName = d.matchName o.matchRegion = d.matchRegion + o.matchSeaweedId = d.matchSeaweedId o.person = d.person o.preemptivePerson = d.preemptivePerson o.pricePkWeekly = d.pricePkWeekly diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy index 40748a9fd..9745c7074 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/region/MapAreaUpdate.groovy @@ -1,17 +1,79 @@ package com.fangjia.script.gj.region +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql class MapAreaUpdate { static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { def task = new MapAreaUpdate() - task.run() + //task.run() + task.checkCcbRegion() } + def checkCcbRegion(){ + def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\建行最终区域编码.xlsx",0,1) + def citys = CcbCommonUtils.citys + def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] + def fjw_citys = [] + for(def d : datas){ + if(!fjw_citys.contains(d['A'])){ + fjw_citys << d['A'] + } + } +// for(def fjwCity : fjw_citys){ +// if(!citys_252.contains(fjwCity) && !citys.contains(fjwCity)){ +// println fjwCity +// } +// } + + for(def newCity : citys_252){ + if(!fjw_citys.contains(newCity)){ + println fjwCity + } + } + + +// //是否全在之前92个城市中 +// def ccb_92_citys = getCcb92City(sqlInstance) +// for(def m : ccb_92_citys){ +// def isExist = false +// for(def d : datas){ +// if(m.cityId==d['E'].toInteger().toString() && m.distId==d['F'].toInteger().toString()){ +// isExist = true; +// break +// } +// } +// if(!isExist){ +// println m.cityId +" "+ m.distId +" 不存在" +// } +// } + } + + + def getCcb92City(sqlInstance){ + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE from bigdata_district_gj" + def list = [] + sqlInstance.eachRow(sql, { r -> + def m = [:] + m.region = r.DISTNAME + m.cityId = r.CITYID + m.distId = r.DISTID + list << m + }) + return list; + } + + def run(){ def rets = findAllCity() for(def city : rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy new file mode 100644 index 000000000..5312d274a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/RegionCode.groovy @@ -0,0 +1,94 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.core.RegionData +import groovy.sql.Sql + +def newRegionCodeFile = "D:\\数据\\建行\\二期\\行政区处理_20220923.xlsx" + +def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +def specialCityMap = ["内蒙古##兴安":"兴安盟","四川##甘孜":"甘孜州","云南##文山":"文山州","青海##玉树":"玉树州"] + +//ExcelUtils.modify(newRegionCodeFile,{d-> +// if(d['A'] && !d['I']){ +// def city = specialCityMap.get(d["A"]+"##"+d["B"]) +// if(!city){ +// city = RegionData.formatCity(d['B']) +// } +// if(city){ +// def region = RegionData.formatRegion(city, d['E']) +// if(region){ +// d['I'] = city +// d['J'] = region +// } +// } +// } +//}) + +def newDatas = ExcelUtils.read(newRegionCodeFile,0,1) +def newDataMap = [:] +for(def d : newDatas){ + def key = d['I']+"##"+d['J'] + def rets = newDataMap.get(key) + if(!rets){ + newDataMap.put(key, d) + } +} + +def ccbMap = getCcb92City(sqlInstance) +ExcelUtils.modify("D:\\数据\\亲邻\\region_code.xlsx",{d-> + if(!d['C'] && d['D']){ + def dbo = CommonDatautils.findAlCityDetail(d['A']) + if(dbo){ + d['C'] = dbo.province + } + } +// if(!d['D']){ +// def m = newDataMap.get(d['A']+"##"+d['B']) +// if(m){ +// d['C'] = m['A'] +// d['D'] = m['B'] +// d['E'] = m['C'] +// d['F'] = m['D'] +// d['G'] = m['E'] +// } +// def m = ccbMap.get(d['A']+"##"+d['B']) +// if(m){ +// d['D'] = m.city +// d['E'] = m.cityId +// d['F'] = m.distId +// d['G'] = m.region +// } +// } +}) + +def getCcb92City(sqlInstance){ + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE from bigdata_district_gj" + def map = [:] + sqlInstance.eachRow(sql, { r -> + def city = getCityName(r.CITYID) + def region = RegionData.formatRegion(city, r.DISTNAME) + def m = [:] + m.city = city + m.region = r.DISTNAME + m.cityId = r.CITYID + m.distId = r.DISTID + map.put(city+"##"+region, m) + }) + return map; +} + +def getCityName(cityId){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def city = null + cityCodeDataMap.each{cityName, id-> + if(id == cityId){ + city = cityName + } + } + return city; +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 926581423..24eeb3c76 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -647,9 +647,9 @@ class CZDataBatch extends BaseBatch{ } } - if(!priceMap){ - priceMap = findRegionRentPrice(priceMap, city, region, estate_type2) - } +// if(!priceMap){ +// priceMap = findRegionRentPrice(priceMap, city, region, estate_type2) +// } //5室及以上价格小于2000去除 if(priceMap){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 1ae32a425..e76898c7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "我爱我家"// - def citys = findCrawlCity(source, 202207); + def source = "安居客"// + def citys = findCrawlCity(source, 202208); for(def city : citys){ - def count1 = findCount(source, city, 202207); - def count2 = findCount(source, city, 202208); + def count1 = findCount(source, city, 202208); + def count2 = findCount(source, city, 202209); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 886acc0b1..68943db41 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.thirdPriceCrawl +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils @@ -11,7 +14,8 @@ import org.bson.types.ObjectId * 福州农商行 * 将抓取的小区均价同步到第三方均价表中 */ -class SynchDistrictPrice { +@BatchOwner(Watcher.GaoJuan) +class SynchDistrictPrice extends BaseBatch{ def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); @@ -20,16 +24,22 @@ class SynchDistrictPrice { // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - //"365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋" - def esSources = ["搜房":"搜房","我爱我家":"我爱我家","365淘房":"365淘房"] + // "365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋" + def esSources = ["吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] + + static main(args) { - def task = new SynchDistrictPrice() -// task.run() - task.run1() + runBatch(new SynchDistrictPrice(), args); + } + + @Override + public void process(Object args) { + run1() } def run1(){ + println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def citys = CommonUtils.findAllCity() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 149149fd0..99355ab2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202207); - def count2 = findTotalCount(source, 202208); + def count1 = findTotalCount(source, 202208); + def count2 = findTotalCount(source, 202209); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 3d4e855f2..68cd266c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -354,12 +354,13 @@ public class OpenApiClient { params.put("city", city); params.put("address", address); params.put("token", TOKEN); - params.put("__refresh_cache","1") +// params.put("__refresh_cache","1") - String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8") + //192.168.0.100:30115 + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") JSONObject result = JSONObject.fromObject(content); while (result == null) { - result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8") + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/address/standard", params, "utf-8") println("${city} ${address} 程序调用异常") Thread.sleep(6000) } @@ -369,7 +370,7 @@ public class OpenApiClient { println("${city} ${address}") } } catch (Exception e) { - println("${city} ${address} ${e.getMessage()}") + println("[http error] ${city} ${address} ${e.getMessage()}") e.printStackTrace(); } return null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 2f146d11f..fdcf818c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde" + def static GAO_DE_AK = "90010d53a16ce9df544acab297645106" /** * 经纬度转地址