diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy index 55204f53b..631b76733 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/baidu/CityLatlngBox.groovy @@ -13,8 +13,9 @@ import com.mongodb.DBObject */ class CityLatlngBox { - public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018) - public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.219", 27017) + //public static final DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.105", 27018) + DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018) + public static final DBCollection LATLNG_COLL = MongoUtils.getCollectionByName("addrepo", "city_latlng_range", "192.168.0.206", 27020) public static void main(String[] args) { CityLatlngBox sd = new CityLatlngBox(); @@ -22,7 +23,7 @@ class CityLatlngBox { } void process() { - def cities = CommonUtils.findAllCity() + def cities = ["白沙","阿坝州","甘孜州","阿勒泰","临沧","那曲","三沙","喀什","昌吉","迪庆州","普洱","德宏州","湘西","塔城","凉山州","屯昌","山南","海西州","阿拉善盟","乐东","定安","怒江州","黄山","图木舒克","楚雄","临高","乌兰察布","延边","文山州","锡林郭勒盟","陵水","天门","淮北","黔东南","万宁","湘潭","海北州","琼中","吐鲁番","惠州","巴音郭楞","恩施","铜仁","黔南","西双版纳州","潜江","林芝","文昌","仙桃","神农架","和田","黔西南","果洛州","儋州","哈密","大理","东方","日喀则","兴安盟","昌江","澄迈","保亭","济源","五指山","阿拉尔","伊犁","临夏","海南州","琼海","石河子","昌都","五家渠","玉树州","博尔塔拉","东营","铜陵","黄南州","廊坊","嘉峪关","马鞍山","宁波","广安","中山","克孜勒苏","厦门","莱芜","鄂州","东莞","海口","鹤壁","六盘水","漯河","三亚","珠海","深圳","乌海","淮南","北海","双鸭山","襄阳","新余","汕头","克拉玛依","鹰潭","盘锦","镇江","石嘴山","嘉兴","铜川","焦作","芜湖","潮州","萍乡","红河","黄石","辽源","辽阳","枣庄","毕节","许昌","阳泉","自贡","遂宁","景德镇","营口","日照","莆田"]//CommonUtils.findAllCity() for (String city : cities) { println city BOUNDS_COLL.remove(new BasicDBObject('city', city)) @@ -70,13 +71,13 @@ class CityLatlngBox { int row = 0 int inx = 0 def buffer = [] - while(seedLatBD < cityBoundBD.getMaxLat()) { + while(seedLatGD < cityBoundGD.getMaxLat()) { int col = 0 Bounds boundsBD = null; Bounds boundsQQ = null; Bounds boundsGD = null; - while (seedLngBD < cityBoundBD.getMaxLng()) { + while (seedLngGD < cityBoundGD.getMaxLng()) { double[] espBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 90) // 右侧OFFSET处的经纬度 double[] wnpBD = GeoUtils.getLatLon(seedLngBD, seedLatBD, offset, 360) // 上方OFFSET处的经纬度 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index e6b9defe3..920c0d4fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ println city +" "+rows.size() } def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] - FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.dat",headers, "utf-8", false); + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false); println "txt 生成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index 08c478e6d..c3affa2bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{ def rets = [] rets << headers rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.dat",headers, "utf-8",false); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); } def createDatas(rows){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index 8a6e106ca..39a26972b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -65,9 +65,9 @@ class BigdataPoiBatch extends BaseBatch{ def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] - def fileName = "/home/goojia/ccb/bigdata_poi.txt" + def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_POI_GJ.dat" } FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); println "txt 生成" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 2124be80f..ad48d4304 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -107,8 +107,8 @@ class CcbCommonUtils { alias << dist.name } - if(dist.name.endsWith("弄小区")){ - def name = dist.name.replace("弄小区","弄") + if(dist.name.endsWith("弄小区") || dist.name.endsWith("小区") ){ + def name = dist.name.replace("弄小区","弄").replace("小区","") if(!alias.contains(name)){ alias << name } @@ -123,10 +123,10 @@ class CcbCommonUtils { def rets = new HashSet() def lists = [] for(def d : dbcursor){ - if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ - //def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") - //if(!rets.contains(key)){ -// rets << key +// if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key d.dist_id = dist._id.toString() d.address = dist.address d.master_name = dist.name @@ -136,8 +136,8 @@ class CcbCommonUtils { d.property_life = dist.property_life d.estate_type2 = dist.estate_type2 lists << d - //} - } + } +// } } return lists } @@ -229,7 +229,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "海昌欣城") +// q.put("name", "东特金星公馆") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy index f340f72ab..1dbc5cd4d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy @@ -43,7 +43,9 @@ class DistrictDelete { static main(args) { def task = new DistrictDelete() // task.run() - task.gzRun(); +// task.gzRun(); +// task.deleteDistrict() + task.deleteCommPrice() } def run(){ @@ -211,9 +213,9 @@ class DistrictDelete { def gzRun(){ def index = 0 - ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d-> - if(d['U'] && d['U'].toString().toDouble().toInteger()==1){ - def id = d['E'] + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\要删除小区 (5).xlsx",{d-> + if(d['G']){ + def id = d['G'] BasicDBObject q=new BasicDBObject('name_cat',1) q.put("m_date", new Date()) DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) @@ -223,4 +225,38 @@ class DistrictDelete { },0) } + def deleteDistrict(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d-> + if(d['G']){ + def id = d['G'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + deleteAllRefDistrict(id) + } + },0) + } + + def deleteAllRefDistrict(id){ + def tables = ["bigdata_forsale_gj","avm_evaluation_item_gj","community_base_price_gj","community_poi_gj","community_sale_gj","community_sale_tmp","community_test_price_tmp"] + for(def tableName : tables){ + deleteByCommid(tableName, id) + } + } + + def deleteCommPrice(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\均价删除.xlsx",{d-> + if(d['B']){ + def id = d['B'] + println id + deleteByCommid("community_base_price_gj",id) + } + },0) + } + + def deleteByCommid(tableName, commid){ + def sql = "delete from "+tableName+" where commid='${commid}'"; + sqlInstance.execute(sql) + println "删除:"+tableName+" "+commid + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index 3f73a32d6..b64fdcd1a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -17,7 +17,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["深圳"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] for (def city : citys) { @@ -34,8 +34,8 @@ class DistrictGuidePriceBatch extends BaseBatch{ } } -// def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] -// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\district_guied_price.dat", headers, "utf-8", false); + def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_GUIDE_PRICE_GJ.dat", headers, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index 7b40980e1..afae30ba4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -29,8 +29,8 @@ class EvaluateRateBatch extends BaseBatch{ def rets = [] rets << headers rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.dat",headers, "utf-8", false); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\EVALUATE_RATE_GJ.xlsx", rets, "数据") + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\EVALUATE_RATE_GJ.dat",headers, "utf-8", false); } def createDatas(def datas){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 4c16c191d..7f6cf3fd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,18 +25,24 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportGuidedPrice() -// exportCommunityGuidedPrice() + exportCommunityGuidedPrice() // exportBigdataDeal() // exportThirdPartyPrice() // exportBigdataCommunity() - exportBigdataForsale() +// exportBigdataForsale() // exportAvmEvaluationGJNB() // exportAVMEvaluationItemGJNB() // exportCommunityBasePrice() + +// exportCommunityBasePriceGj() +// exportCityPrice() +// exportRegionPrice() +// exportLastFile() } + def exportEvaluationRateData(){ def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; def allRows = [] @@ -213,7 +219,7 @@ class ExportOracleData { def exportGuidedPrice(){ - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where cityId in (350200,150100)"; + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=TO_DATE('2021-08-01', 'yyyy-MM-dd') and dealdate def data = [] @@ -237,20 +243,20 @@ class ExportOracleData { def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" def dataCodeFileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.ok" } def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "GUIDED_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202107") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") println "GUIDED_PRICE_GJ 上传Ftp完成......" } def exportCommunityGuidedPrice(){ - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='厦门' or cityname='呼和浩特'"; + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where cityname='西安'"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -273,16 +279,16 @@ class ExportOracleData { def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202107\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.ok" } def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, "202108") - CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + CcbFtpUtils.ftpUpload(fileName, "202107") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202107") println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" } @@ -726,5 +732,88 @@ class ExportOracleData { println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" } + def exportCommunityBasePriceGj(){ + + } + + def exportCityPrice(){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where (cityId=350200 or cityId=150100) and year_month>=202108"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_COMMUNITY_CITY_PRICE_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "CITY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "CITY_PRICE_GJ 上传Ftp完成......" + } + + def exportRegionPrice(){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where (cityId=350200 or cityId=150100) and year_month>=202108"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + def dataCodeFileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\202108\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.ok" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + println "DISTRICT_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, "202108") + CcbFtpUtils.ftpUpload(dataCodeFileName, "202108") + println "DISTRICT_PRICE_GJ 上传Ftp完成......" + } + + def exportLastFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + + def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curMonth}\\${curDate}.ok" + } + CcbFtpUtils.ftpUpload(dataCodeFileName, "${curMonth}") + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 68eb41ed3..351473f24 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -71,7 +71,7 @@ class GuiDingPriceBatch extends BaseBatch{ } def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", false); + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); // def rets = [] // rets << headers diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LocationUpdate.groovy new file mode 100644 index 000000000..a34e7f2d7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LocationUpdate.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +class LocationUpdate { + + static main(args) { + def task = new LocationUpdate(); + task.run() + } + + def run(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度\\经纬度人工核实结果_0923.xlsx",{d-> + index++ + if(d['C']){ + + println index +" "+d['A']+" "+d['B']+" "+d['C'] + def region = RegionData.formatRegion(d['A'], d['B']) + if(d['H']){ + def status = d['H'].toString().toDouble().toInteger() + if(status==1){ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + def gmap = GeocodeUtils.getGaodeToBaidu(d['F'].toString().toDouble(), d['G'].toString().toDouble()) + def lng2 = null + def lat2 = null + if(gmap){ + lng2 = (gmap.get("lng")*10000000000).longValue() + lat2 = (gmap.get("lat")*10000000000).longValue() + } + + BasicDBObject q=new BasicDBObject() + q.put("m_date", new Date()) + q.put("lat2", lat2) + q.put("lng2", lng2) + q.put("lng", (d['F'].toString().toDouble()*10000000000).longValue()) + q.put("lat", (d['G'].toString().toDouble()*10000000000).longValue()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + + }else{ + println d['A']+" "+d['B']+" "+d['C']+" 没找到小区" + } + }else{ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + }else{ + println d['A']+" "+d['B']+" "+d['C']+" 没找到小区" + } + } + + }else{ + def obj = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") + if(obj){ + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',q)) + }else{ + println d['A']+" "+d['B']+" "+d['C']+" 没找到小区" + } + + } + + } + + }) + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 3c3af2392..26db4158d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{ results << headers results.addAll(rows) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING.xlsx", results, "数据") - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING.dat",headers, "utf-8", false); + ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据") + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 18dc9f380..f9ac2e3fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -3,6 +3,8 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point import com.fangjia.script.util.DbUtils @@ -14,6 +16,11 @@ import com.mongodb.DBCollection import groovy.sql.Sql import org.bson.types.ObjectId +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong import java.util.regex.*; @@ -42,7 +49,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw // println city +" "+count //} -//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx" +//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) //def dealMap = [:] //for(def d : datas){ @@ -59,7 +66,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw -def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx" +//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) //def dealMap = [:] @@ -83,7 +90,7 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // if (city) { // def cityid = cityCodeDatas.get(city) // def total = findCount(cityid, sqlInstance) -// d['M'] = total +// d['O'] = total // } //},0) @@ -128,13 +135,26 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // } //},1) -//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx" -def dealMap = findDealCount(sqlInstance) -ExcelUtils.multiThreadingModify(fileName,{d-> - if(d['C']){ - d['K'] = dealMap.get(d['C']) +//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx" +//def dealMap = findDealCount(sqlInstance) +//ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['A']){ +// d['U'] = dealMap.get(d['A']) +// } +//},0) + + +def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx" +def datas = ExcelUtils.read(fileName, 0, 1) +def dealMap = [:] +for(def d : datas){ + dealMap.put(d['A'], d['B']) +} +ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d-> + if(d['A']){ + d['W'] = dealMap.get(d['A']) } -},2) +},0) //def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" @@ -170,7 +190,7 @@ ExcelUtils.multiThreadingModify(fileName,{d-> def findDealCount(sqlInstance){ - def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate @@ -204,7 +224,7 @@ def findCount(cityid, sqlInstance){ } def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and "+field+" is not null"; + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; def count = 0 sqlInstance.eachRow(sql,{r-> count = r.count @@ -353,3 +373,5 @@ def getStreetInfo(def city, def glocation){ + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy new file mode 100644 index 000000000..c0833369a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy @@ -0,0 +1,81 @@ +package com.fangjia.script.ccbevaluation + +import groovy.sql.Sql + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class TestRepeat { + + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new TestRepeat() + task.run() + } + def run(){ + + } + + + def checkUpdateCount(){ + def newHouses = readTxt("C:\\Users\\gaojuan\\Desktop\\BIGDATA_FORSALE_GJ.dat") + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(newHouses.size()) + println " 房源量:"+newHouses.size() + + def retMap = new ConcurrentHashMap() + for(def i=0;i + obj = r + }); + return obj + } + + + def readTxt(def file){ + def lines = org.apache.commons.io.FileUtils.readLines(new File(file)) + def rets = []; + for(line in lines) { + def arrs = line.split("|@|") + def map = [:] + map.houseId = arrs[1] + map.commId = arrs[6] + map.pdate = arrs[10] + rets << map + } + return rets; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8fadf19e0..1e8ca6a82 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -20,9 +20,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202107//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202105 - def endDate = 202107 + def sDate= 202108//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202106 + def endDate = 202108 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -53,13 +53,13 @@ class ThirdPartyPriceBatch extends BaseBatch { - def fileName = "/home/goojia/ccb/third_party_price.dat" + def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat" } - def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); } @@ -67,6 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ def rets = [] for(def d : dists){ + def third = thirdPriceMap.get(d.region+"##"+d.name) if(third){ def data = distCodeMap.get(d.region) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index c2378636d..a35a99a4e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -1,12 +1,22 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong /*** * 建行成交抓取统计 @@ -21,23 +31,34 @@ class TradeStatLog { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new TradeStatLog() - task.run() +// task.run() // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) +// task.averageStat() + task.exportTrade() +// task.deleteDistrict() + } def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","6月总量","7月总量","8月总量","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] - def startDate = "2021-05-01" - def date = "2021-06-01" - def endDate = "2021-07-01" - def newDate = "2021-08-01" + //def startDate = "2021-05-01" + def startDate = "2021-06-01" + def date = "2021-07-01" + def endDate = "2021-08-01" + def newDate = "2021-09-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -64,7 +85,7 @@ class TradeStatLog { ret << lastMonthCount ret << curMonthCount ret << nearMonthCount - if(lastMonthCount){ + if(curMonthCount){ def a = (nearMonthCount-curMonthCount)*100/curMonthCount def pre = DataHandleUtils.getDoubleRound(2, a) ret << pre @@ -87,7 +108,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx", rets, "数据") println "表格生成完成" } @@ -103,7 +124,7 @@ class TradeStatLog { rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14] println rets.get(rets.size()-1) } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计0927.xlsx", rets, "数据") println "表格生成完成" } @@ -129,7 +150,7 @@ class TradeStatLog { def getThirdPartyPricePrecent(def city){ - def sql = "SELECT * FROM third_party_price WHERE cityname = '"+city+"'"; + def sql = "SELECT * FROM third_party_price_gj WHERE cityname = '"+city+"' and year_month=202108"; def count0 = 0 def count1 = 0 @@ -299,5 +320,333 @@ class TradeStatLog { } + def averageStat(){ + def citys = ["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"] + + def map = [:] + for(def city : citys){ + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]); + def rets = findThirdPriceByDist(districts) + map.put(city, rets.size()) + map.put(city+"-count", districts.size()) + println districts.size()+" "+ rets.size() + } + println map + + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\估值测试数据覆盖率占比.xlsx",{d-> + def city = d['A'] + def count = map.get(city) + def distCount = map.get(city+"-count") + d['H'] = count + d['J'] = distCount + + def pre = DataHandleUtils.getDoubleRound(2, count*100.0/distCount) + d['I'] = pre + },0) + } + + + def findThirdPriceByDist(districts){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + for(def i=0;iendPrice){ +// reason = "价格过滤" +// } + if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){ + reason = "小区过滤" + } + + if(!reason){ + def key=t.city+"##"+dist.region+"##"+dist.name+"##"+t.area+"##"+t.avg_price+"##"+DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd") + if(details.contains(key)){ + reason = "数据重复" + } + } + } + + + def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd") + //rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type] + rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason] + } + } + println city +" "+rets.size() + } + + ExcelUtils.write("d:\\数据\\成交未入库记录(7月全).xlsx", rets,"数据") + } + + def deleteDistrict(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d-> + if(d['G']){ + def id = d['G'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + deleteAllRefDistrict(id) + } + },0) + } + + def deleteAllRefDistrict(id){ + def tables = ["bigdata_community_gj"] + for(def tableName : tables){ + deleteByCommid(tableName, id) + } + } + + def deleteByCommid(tableName, commid){ + def sql = "delete from "+tableName+" where commid='${commid}'"; + sqlInstance.execute(sql) + println "删除:"+tableName+" "+commid + } + + def getYouDaData() { + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd")) + def endDate = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") + def q = new BasicDBObject() +// q.put("premisesName","共富一村") + q.put("contractTime", new BasicDBObject('$gte', startDate));//.append('$lt', endDate) + def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000) + def trades = [] + for(def db : dbcursor){ + def map = [:] + map._id=db._id.toString() + map.city = "上海" + map.region = db.region + if(db.districtName && db.districtName!="null"){ + map.district_name=db.districtName + }else if(db.address){ + map.district_name=db.address + } + + map.area = db.area + map.trade_date = db.contractTime + map.avg_price= db.price + map.estate_type = db.houseType + trades << map + } + return trades + } + + def findDealIds(city){ + def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate + if(!ids.contains(r.dealid)){ + ids << r.dealid + + def region = RegionData.formatRegion(r.cityname, r.distname) + def key=r.cityname+"##"+region+"##"+r.commname+"##"+r.buildarea.toDouble()+"##"+r.dealunitprice+"##"+DateUtils.formatDateByPattern(r.dealdate,"yyyy-MM-dd") + if(!detailInfos.contains(key)){ + detailInfos << key + } + } + }); + def rMap = [:] + rMap.ids = ids + rMap.detail = detailInfos + return rMap + } + + + public static def findTradesDetail(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) +// q.put("district_name","百合花园") + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets + } + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"@"+d.district_name, d.avg_price) + } + return map + } + + def getAllDistrict(city){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"]) + def map = [:] + for(def d : dists){ + map.put(d.city+"##"+d.region+"##"+d.name, d._id.toString()) + def alias = d.alias + if(alias){ + for(def name : alias){ + map.put(d.city+"##"+d.region+"##"+name, d._id.toString()) + } + } + } + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy index a47028b79..cb189d420 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateHouseRegion.groovy @@ -1,13 +1,18 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.ExcelUtil import com.fangjia.core.util.HttpRequest import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import net.sf.json.JSONObject +import org.bson.types.ObjectId import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet @@ -16,6 +21,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern class UpdateHouseRegion { @@ -37,11 +43,89 @@ class UpdateHouseRegion { static main(args) { def task = new UpdateHouseRegion() - task.updateSeawaterRegion() +// task.updateSeawaterRegion() // task.updateDealRegion() // task.updateThirdPartyRegion() + task.updateNoMatchDealRegion() +// task.addAlias() +// task.addFenQi() } + def addFenQi(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"]) + addFenQiAlias(dists) + println city +" 处理完成" + } + } + + def addFenQiAlias(dists){ + for(def dist : dists){ + def s1 = DataUtil.match(dist.name, Pattern.compile("[一|二|三|四|五|六|七|八|九|十|十一|十二|十三|十四|十五|十六|十七|十八|\\w+](期|区)\$")) + if(s1){ + def newName = dist.name.replace("Ⅰ","1").replace("Ⅱ","2").replace("A区","a区").replace("B区","b区").replace("C区","c区").replace("D区","d区").replace("E区","e区").replace("F区","f区").replace("一期","1期").replace("二期","2期").replace("三期","3期").replace("四期","4期").replace("五期","5期").replace("六期","6期").replace("七期","7期").replace("八期","8期").replace("九期","9期").replace("十期","10期").replace("十一期","11期").replace("十二期","12期") + if(newName!=dist.name){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(!alias.contains(newName)){ + alias << newName + def udata = new BasicDBObject() + udata.put("alias", alias) + DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) + println dist.city +" "+dist.name+" "+dist.alias+" 新:"+alias + } + } + } + } + } + + def addAlias(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district",["city","region","name","alias"]) + addDistricts(dists) + println city +" 处理完成" + } + } + + def addDistricts(dists){ + for(def dist : dists){ + def alias = dist.alias + if(alias){ + def oldCount = alias.size() + def newAlias = [] + for(def aliasName : alias){ + if(aliasName.contains("·") || aliasName.contains(".")){ + def newName = aliasName.replace("·","").replace(".","") + newAlias << aliasName + if(!newAlias.contains(newName)){ + if(newName!=dist.name){ + newAlias << newName + } + } + }else{ + if(!newAlias.contains(aliasName)) { + newAlias << aliasName + } + } + } + def newCount = newAlias.size() + if(oldCount && alias!=newAlias){ + def udata = new BasicDBObject() + udata.put("alias", newAlias) + DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) + println dist.city +" "+dist.name+" "+dist.alias+" 新:"+newAlias + } + + + } + } + } + + def updateThirdPartyRegion(){ def city = "东莞" def rets = findThirdPartyPrice(city) @@ -121,6 +205,32 @@ class UpdateHouseRegion { latch.await() } + def updateNoMatchDealRegion(){ + ExcelUtils.multiThreadingModify("D:\\数据\\成交未入库记录(7月全).xlsx",{d-> + if(!d['J']){ + def city = d['B'] + def region = d['C'] + def name = d['D'] + def rets = CommonDatautils.findDistrictsByName(city, name, "district") + if (!rets) { + rets = CommonDatautils.findDistrictByAlias(city, null, name, "district") + } + + if (rets.size() == 1) { + def dist = rets[0] + if (region != dist.region) { + def udata = new BasicDBObject() + udata.put("region", dist.region) + DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + d['J'] = "原数据区域修改("+dist.region+")" + println d['A']+" "+region +" "+dist.region + } + } + } + + },0) + } + def updateSeawaterRegion(){ def city = "东莞" def rets = findHouseSource(city, 20190701, 20211001) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index c61861c99..ad5a8babc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv") def count0 = 0 def count1 = 0 def count2 = 0 @@ -135,7 +135,7 @@ rets << ["9 def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计") def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" - +println s def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" DingDingMessageUtil.sendTextMessage(access_token, s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy index c322efc25..45c2f60a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenUserBatch.groovy @@ -45,18 +45,20 @@ class OpenUserBatch { } def addShenZhenFjtUser(){ - def datas = ExcelUtils.read("D:\\数据\\农行\\肇庆农行支行房价通账号.xlsx",0,1) + def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1) def i = 0; for(def data : datas){ i++; - def id = 252580 + i; - def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+data[2]+"', '"+data[3]+"', '16', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2021-12-10 00:00:00', '0', '', NULL)"; + def id = 252608 + i; + BigDecimal bd = new BigDecimal(data[3].toString()); + String mobile = bd.toPlainString(); + def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)"; onLineSqlInstance.execute(sql); - def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[1]+"', '"+data[0]+"', NULL, NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王辰岑')"; + def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')"; onLineSqlInstance.execute(sql2) - def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'肇庆','肇庆:')" + def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')" onLineSqlInstance.execute(sql3) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy new file mode 100644 index 000000000..c90064d97 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbXykData.groovy @@ -0,0 +1,18 @@ +package com.fangjia.script.gj.ccb + +import com.fangjia.script.commData.CommonDatautils + +class CcbXykData { + static main(args) { + def task = new CcbXykData() + task.run() + } + + def run(){ + def districts = CommonDatautils.findAllDistrictByCity("上海","district",["city","region","name","estate_charge"]) + for(def d : districts){ + + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index e7232a3c8..7d7f43a1c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -9,6 +9,14 @@ class FileUtils { // file.delete(); // } + if (!file.getParentFile().exists()) { + boolean result = file.getParentFile().mkdirs(); + if (!result) { + System.out.println("创建目录失败"); + } + } + + OutputStream out = new FileOutputStream(fileName, isAppend); def sb = new StringBuffer(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy new file mode 100644 index 000000000..0ad5dbfcf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.gj.location + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.NameProcess +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet + +class DistrictLocationUpdate { + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static main(args) { + def task = new DistrictLocationUpdate() + task.run() + } + def run(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"]) + + + updateLocation(city, dists); + + } + + } + + def queryPoiMap(def city){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "'") + sqlBuffer.append(" and source = '高德'") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def retMap = [:] + while (rs.next()) { + def map = [:] + map.city = rs.getString("cityname") + map.region = rs.getString("adname") + map.name = rs.getString("name") + map.address = rs.getString("address") + map.type = rs.getString("type") + def location = rs.getObject("location") + if(location){ + map.lng = location.x + map.lat = location.y + } + + def name = NameProcess.nameStandard(map.city, map.region, map.name) + if(name){ + retMap.put(map.region+"##"+name, map) + } + } + return retMap; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy index 4313382e5..d60d5f02c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy @@ -521,6 +521,10 @@ class GaoDeUtils { return KeyList.get(count); } + static void main(def a){ + println search("如东县大豫镇豫东村委会","南通") + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy new file mode 100644 index 000000000..7d52d6a3b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/MatchDistrict.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.RegionData +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +class MatchDistrict { + static main(args) { + def task = new MatchDistrict() + task.run() + } + def run(){ + def filePath = "D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx" + def city = "深圳" + ExcelUtils.multiThreadingModify(filePath, { data -> + if(data['A']){ + def region = RegionData.formatRegion(city, data['A']) + if(region){ + def rMap = SeaweedDataMatchProcess.doMatch(city, data['A'], data['C'], null); + if(rMap){ + data['E'] = rMap.city + data['F'] = rMap.region + data['G'] = rMap.name + } + } + } + }, 0); + } +} + +//{ +// "_id" : ObjectId("607963dc5bb87e2a18f4dde6"), +// "guiding_type" : NumberInt(0), +// "estate_type" : "普通住宅", +// "house_type" : NumberInt(0), +// "c_date" : ISODate("2021-04-16T10:15:56.651+0000"), +// "guiding_person" : "深圳房管局", +// "guiding_date" : ISODate("2021-04-10T16:00:00.000+0000"), +// "category" : "district", +// "guiding_value" : NumberInt(48300), +// "name" : "中粮云景国际", +// "city" : "深圳", +// "region" : "光明" +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index aa345e412..91498750a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -1,33 +1,55 @@ package com.fangjia.script.gj.test +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo +import groovy.sql.Sql import java.text.SimpleDateFormat //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); //Date date = new Date(1617211269868); //print dateFormat.format(date); -DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info'); -def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx" -ExcelUtils.modify(path,{d-> - def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"]) - d['G'] = districts.size() +//DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info'); +//def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx" +//ExcelUtils.modify(path,{d-> +// def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"]) +// d['G'] = districts.size() +// +// def q = new BasicDBObject() +// q.put("city", d['B']) +// q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08")) +// def obj = CITY_INFO_CONN.findOne(q); +// if(obj){ +// d['H'] = obj.price_coverage*100 +// } +//}) - def q = new BasicDBObject() - q.put("city", d['B']) - q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08")) - def obj = CITY_INFO_CONN.findOne(q); - if(obj){ - d['H'] = obj.price_coverage*100 +def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); + +def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; +def countMap = [:] +onLineSqlInstance.eachRow(sql,{r-> + def addTime = r.visit_time + def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") + def ips = countMap.get(month) + if(!ips){ + ips = new HashSet(); + countMap.put(month, ips) } + ips << r.ip +}); + +countMap.each {k,v-> + println k+" "+v.size() +} + + - -}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy new file mode 100644 index 000000000..7f49b80b0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/test/RegionalismCodeStat.groovy @@ -0,0 +1,58 @@ +package com.fangjia.script.test + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class RegionalismCodeStat { + + DBCollection REGIONALISM_CODE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018) + + + static main(args) { + def task = new RegionalismCodeStat() + task.run() + } + + def run(){ + def provinces = ["上海市","北京市","天津市","重庆市","福建省","贵州省","湖南省","广东省","湖北省","安徽省","四川省","江苏省","吉林省","河北省","河南省","海南省","江西省","云南省","甘肃省","山东省","陕西省","浙江省","青海省","辽宁省","黑龙江省","山西省","西藏自治区","新疆维吾尔自治区","宁夏回族自治区","内蒙古自治区","广西壮族自治区"] +// def provinces = ["宁夏回族自治区"] + provinces.each {v-> + def sets = findCount(v) + println v +" "+ sets.size() + } + + } + + def findCount(v){ + def q = new BasicDBObject() + q.put("province", v) + def dbcursor = REGIONALISM_CODE_COLL.find(q).batchSize(5000) + def set = new HashSet() + for(def obj : dbcursor){ +// if(obj.city=="省直辖县级行政区划" || obj.city=="自治区直辖县级行政区划"){ +// //set << obj.region +// println "【"+obj.region+"】 过滤" +// continue +// }else{ +// if(obj.region && !set.contains(obj.region)){ +// set << obj.region +// println "【"+obj.region+"】 add" +// } +// } + + if(obj.juweihui){ + def k = obj.street+"##"+obj.juweihui + if(!set.contains(k)){ + set << k +// println "【"+k+"】 add" + } + } + } + +// set.each {k-> +// println k +// } + return set; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 0ba834ad5..7ae95777c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,7 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产"] - def sourceMap = ["zhuge.com":"诸葛找房"] + def sourceMap = ["cityhouse.cn":"城市房产"]//["zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 10283440a..e299c60f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202107); - def count2 = findTotalCount(source, 202108); + def count1 = findTotalCount(source, 202108); + def count2 = findTotalCount(source, 202109); println source +" "+count1 +" "+count2 } }