From cd24e28b71bb0d32c2f127cef91d288b1197eac1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Apr 2023 18:44:17 +0800 Subject: [PATCH 01/19] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index ae3119029..3f04740d6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" // def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] - def esSources = ["搜房":"搜房"] + def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] static main(args) { From 40b176a5f7de08e8562787f8e3239d4211643d2d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 12 Apr 2023 14:19:13 +0800 Subject: [PATCH 02/19] =?UTF-8?q?=E6=96=B0=E5=A2=9Epoi=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/dataAna/CityInfoAnaBatch.groovy | 79 ++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index d4d8007e7..5086f52d6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -50,6 +50,7 @@ class CityInfoAnaBatch extends BaseBatch { final static DBCollection HOUSE_LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia", "house_loudong", MongoUtils.MongoDB_Domain_mongo); final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo); final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000); + final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo) static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616') final static DSCollection SEAWEED = DbUtils.seaweed public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) @@ -62,7 +63,7 @@ class CityInfoAnaBatch extends BaseBatch { car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure', lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', - alias:'alias',addr_alias:'addrAlias' + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea' ] public static final def OFFICE_FIELDS_MAP = [ name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', @@ -85,6 +86,7 @@ class CityInfoAnaBatch extends BaseBatch { final static def COUNT_CITYS = ["滁州","黄山","马鞍山","芜湖","宿州","宣城","北京","白银","定西","甘南","金昌","兰州","临夏","陇南","平凉","庆阳","天水","武威","张掖","东莞","佛山","广州","惠州","江门","梅州","清远","深圳","玉林","安顺","贵阳","铜仁","三亚","海口","承德","邯郸","衡水","廊坊","秦皇岛","石家庄","开封","洛阳","大庆","鄂州","黄冈","十堰","武汉","宜昌","常德","长沙","常州","淮安","连云港","南京","南通","苏州","泰州","无锡","宿迁","徐州","盐城","扬州","镇江","景德镇","九江","上饶","鹰潭","抚顺","辽阳","盘锦","沈阳","铁岭","营口","呼和浩特","石嘴山","银川","海东","西宁","滨州","德州","济南","济宁","青岛","潍坊","淄博","太原","宝鸡","渭南","西安","咸阳","上海","成都","达州","德阳","乐山","凉山州","泸州","眉山","绵阳","南充","内江","遂宁","雅安","宜宾","资阳","自贡","天津","丽江","昭通","杭州","湖州","嘉兴","金华","丽水","宁波","衢州","绍兴","台州","温州","舟山","重庆"]; + final static def DIST_FACILITY_LIST = ["bank", "hospital", "bus", "shop", "university", "middle_school", "primary_school", "kindergarten", "food_shop", "supermarket","atm","school","subway"] /** * 统计结果接收邮箱列表 */ @@ -116,7 +118,9 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> - CITYS << k.toString(); + if(k.toString()=="上海"){ + CITYS << k.toString(); + } } // 所有城市的人口信息 @@ -565,6 +569,8 @@ class CityInfoAnaBatch extends BaseBatch { entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4)) } entity.put('district_detail', districtDetail(disCount, cityName)); + + entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName)); } else { entity.put('district_detail', [:]); } @@ -1079,6 +1085,75 @@ class CityInfoAnaBatch extends BaseBatch { districtDetail.put('rentPrice', roundDouble(pc, disCount)); return districtDetail; } + + def districtFacilitysDetail(def disCount, def cityName){ + DBObject query = new BasicDBObject(); + query.put('city', cityName); + query.put('cat', 'district'); + query.put('visible', 0); + + DBObject fields = new BasicDBObject() + fields.put("city", 1) + fields.put("region", 1) + fields.put("name", 1) + + def facilityMap = findFacilityByCity(cityName) + DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) + Iterator iter = cur.iterator() + def countMap = [:] + + while(iter.hasNext()) { + def dbo = iter.next() + def facilityCountMap = facilityMap.get(dbo.city+"##"+dbo.region+"##"+dbo.name) + if(facilityCountMap){ + DIST_FACILITY_LIST.each() { + def count = facilityCountMap.get(it) + if (count) { + if(countMap.containsKey(it)) { + def tempCount = countMap.get(it) + 1 + countMap.put(it, tempCount) + }else { + countMap.put(it, 1) + } + } + } + } + } + + def facilityDetail = [:]; + DIST_FACILITY_LIST.each() {it-> + if(countMap.containsKey(it)) { + facilityDetail.put(it, roundDouble(countMap.get(it), disCount)) + }else { + facilityDetail.put(it, 0) + } + } + return facilityDetail; + } + + def findFacilityByCity(city){ + DBObject query = new BasicDBObject(); + query.put('city', city); + DBCursor cur = FACILITY_CONN.find(query).batchSize(2000) + Iterator iter = cur.iterator() + def distMap = [:] + while(iter.hasNext()) { + def dbo = iter.next() + def countMap = [:] + DIST_FACILITY_LIST.each { + def values = dbo.get(it) + def count = 0 + if(values){ + count = values.size() + countMap.put(it, count) + } + } + if(countMap){ + distMap.put(dbo.city+"##"+dbo.region+"##"+dbo.district_name, countMap) + } + } + return distMap; + } double roundDouble(int a, int base) { if(a == 0 || base == 0) { From ae64bc15114ee995dc69608f32e8f111adf7209d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 14 Apr 2023 21:32:39 +0800 Subject: [PATCH 03/19] update --- .../fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 3f04740d6..a254cbdd7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -43,7 +43,7 @@ class SynchDistrictPrice extends BaseBatch{ println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def citys = ["凉山州"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ def rets = findAllEsData(city, esSource, mgSource, mdate) From b5fc0aecc542becb2dab19f253e12500c9f93f9c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 14 Apr 2023 22:58:51 +0800 Subject: [PATCH 04/19] up --- .../com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index ac8449aa8..f7dcc4293 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -124,7 +124,7 @@ class ThirdPriceExportBatch extends BaseBatch{ add = true; } - def tc_price = getLastThirdPrice(o.get("58tc_esf_price"), curPrice) + def tc_price = getLastThirdPrice(o.anjuke_esf_price, curPrice)//58tc_esf_price row << tc_price if(tc_price){ add = true; From 6abf0257ed0b81b12d85a70f4cd99d4da1be2a03 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 17 Apr 2023 16:34:45 +0800 Subject: [PATCH 05/19] =?UTF-8?q?=E5=86=99=E5=AD=97=E6=A5=BC=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/dataAna/CityInfoAnaBatch.groovy | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 5086f52d6..f30447a24 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -66,20 +66,26 @@ class CityInfoAnaBatch extends BaseBatch { alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea' ] public static final def OFFICE_FIELDS_MAP = [ - name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', - all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', - car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure', - facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics' + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure', + facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo',licence:'licence', + sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany',estate_charge:'estateCharge', + floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife', + conditioner_time:'conditionerTime',elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount', + goods_elevator_count:'goodsElevatorCount',network_way:'networkWay',fitment_level:'fitmentLevel',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea', + is_foreign:'foreign',is_register:'register',lighting_ventilation:'lightingVentilation',lobby_height:'lobbyHeight' ] - + public static final def SHOP_FIELDS_MAP = [ name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate', all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany', rent_type :'rentType',lease_term :'leaseTerm',car_site :'carSite',estate_charge :'estateCharge',divisible :'divisible',floor :'floor', finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate', bay_area :'bayArea', property_life :'propertyLife',unified_manage :'unifiedManage',licence :'licence',business_status :'businessStatus', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',bed_kind:'bedKind',floor_info:'floorInfo',car_site_count :'carSiteCount', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics' + ] public static CITY_GRADE_MAP = [1000: "一线" , 1500 : "新一线" , 2000 : "二线" , 3000:"三线" ,4000 : "四线" ,5000 : "五线"] @@ -91,8 +97,7 @@ class CityInfoAnaBatch extends BaseBatch { * 统计结果接收邮箱列表 */ final static def receivers = [ - "puxueli@fangjia.com", - "wangchencen@fangjia.com", + "gaojuan@fangjia.com", "caoyuyan@fangjia.com", "muliangshu@fangjia.com" ] @@ -118,9 +123,9 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> - if(k.toString()=="上海"){ +// if(["上海","深圳","北京","广州","杭州","苏州","重庆","成都"].contains(k.toString())){ CITYS << k.toString(); - } +// } } // 所有城市的人口信息 @@ -150,7 +155,7 @@ class CityInfoAnaBatch extends BaseBatch { allCityDisCount.put(it.name, it.count.intValue()); } } - + // 城市新盘数 def allCityNewDisCount = [:]; disQuery = new BasicDBObject(); @@ -166,7 +171,7 @@ class CityInfoAnaBatch extends BaseBatch { allCityNewDisCount.put(it.name, it.count.intValue()); } } - + // 城市写字楼数 def allCityOfficeCount = [:]; disQuery = new BasicDBObject(); @@ -182,8 +187,8 @@ class CityInfoAnaBatch extends BaseBatch { allCityOfficeCount.put(it.name, it.count.intValue()); } } - - + + // 城市商铺数 def allCityShopCount = [:]; disQuery = new BasicDBObject(); @@ -507,7 +512,7 @@ class CityInfoAnaBatch extends BaseBatch { entity.put('official_population', curInfo.official_population); entity.put('district_avg_population', curInfo.district_avg_population); entity.put('rate', curInfo.rate); - + removeCurInfo(cityName, weekend); } else { def cityHouseStat = allHouseStats.get(cityName); @@ -517,13 +522,13 @@ class CityInfoAnaBatch extends BaseBatch { } else if (cityHouseStat && cityHouseStat.standard_people_count) { entity.put('official_population', cityHouseStat.standard_people_count); } - + if (lastInfo && lastInfo.district_avg_population) { entity.put('district_avg_population', lastInfo.district_avg_population); } else if (cityHouseStat && cityHouseStat.district_avg_population) { entity.put('district_avg_population', cityHouseStat.district_avg_population); } - + if (lastInfo && lastInfo.rate) { entity.put('rate', lastInfo.rate.toDouble()); } else if (cityHouseStat) { @@ -1252,7 +1257,7 @@ class CityInfoAnaBatch extends BaseBatch { DBObject query = new BasicDBObject(); query.put('city', cityName); query.put('cat', 'office'); - query.put('visible', 0); + query.put('status', 0); DBObject fields = new BasicDBObject() OFFICE_FIELDS_MAP.each() {key,skey-> @@ -1262,7 +1267,7 @@ class CityInfoAnaBatch extends BaseBatch { DSCursor cur = SEAWEED.find(query, fields).batchSize(2000) Iterator iter = cur.iterator() def countMap = [:] - + println cur.count() while(iter.hasNext()) { def dbo = iter.next() OFFICE_FIELDS_MAP.each() {key,skey-> From a397ac3f5cb301df1c4b7a7bd1197baba1cbb954 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 18 Apr 2023 19:04:04 +0800 Subject: [PATCH 06/19] update --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 78e552ea5..db2c71d84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319, houseIdsfilter); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230318, 20230419, houseIdsfilter); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index f30447a24..45da16ec1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -63,7 +63,7 @@ class CityInfoAnaBatch extends BaseBatch { car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter', start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure', lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure', - alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea' + alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount' ] public static final def OFFICE_FIELDS_MAP = [ name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', From aef91d7af61406458c1c7df3873fffb6e7889a22 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Apr 2023 10:57:52 +0800 Subject: [PATCH 07/19] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E6=9B=B4?= =?UTF-8?q?=E6=94=B9=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 94d42d3a3..7f6dd3e59 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202302//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202212 - def endDate = 202302 + def sDate= 202303//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202301 + def endDate = 202303 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From 2e3744df77ae57739a0dcf1151bc59d2eef2ae3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 19 Apr 2023 18:15:05 +0800 Subject: [PATCH 08/19] Update Jenkinsfile --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 37a7f1e92..bcc70fefc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ pipeline { agent any parameters { - choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'], description: '璇烽�夋嫨鍙戝竷鐨勯」鐩�') + choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'], description: '璇烽�夋嫨鍙戝竷鐨勯」鐩�') } tools { jdk 'JDK8' @@ -17,7 +17,7 @@ pipeline { stage('Maven Build') { steps { script { - def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'] for (def project : projects) { if (params.PROJECT_NAME == project) { dir("deliverables/${project}") { @@ -34,7 +34,7 @@ pipeline { steps { script { def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] - def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] + def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static', 'geluke':"geluke"] for (def project : projects) { if (params.PROJECT_NAME == project) { dir("projects/${imagesName[project]}") { @@ -54,7 +54,7 @@ pipeline { steps { script { def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] - def k8s_config_map = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] + def k8s_config_map = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static', 'geluke':"geluke"] for (def project : projects) { if (params.PROJECT_NAME == project) { dir("deliverables/k8s") { From 2705cbdc9b3cf2c35292bdb91912dfe95a282bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 19 Apr 2023 18:58:21 +0800 Subject: [PATCH 09/19] Update Jenkinsfile --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bcc70fefc..28ae94ed0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,7 +22,7 @@ pipeline { if (params.PROJECT_NAME == project) { dir("deliverables/${project}") { withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { - sh 'mvn clean package -Dmaven.test.skip=true' + sh 'mvn package -Dmaven.test.skip=true' } } } @@ -33,7 +33,7 @@ pipeline { stage('Docker Build') { steps { script { - def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'] def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static', 'geluke':"geluke"] for (def project : projects) { if (params.PROJECT_NAME == project) { @@ -53,7 +53,7 @@ pipeline { stage('K8S Deploy') { steps { script { - def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'] def k8s_config_map = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static', 'geluke':"geluke"] for (def project : projects) { if (params.PROJECT_NAME == project) { From 74d80db906d203a7ddc75670986c5bdf7590909d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 19 Apr 2023 19:30:44 +0800 Subject: [PATCH 10/19] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 28ae94ed0..b8b0cf0b7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -22,7 +22,7 @@ pipeline { if (params.PROJECT_NAME == project) { dir("deliverables/${project}") { withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { - sh 'mvn package -Dmaven.test.skip=true' + sh 'mvn clean package -Dmaven.test.skip=true' } } } From 4602691587ed0926c621d1bb91f93342406794ba Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Apr 2023 14:49:14 +0800 Subject: [PATCH 11/19] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E7=AC=AC=E4=B8=89?= =?UTF-8?q?=E6=96=B9=E5=8E=BB=E9=99=A4=E6=B7=B1=E5=9C=B3=E6=9C=89=E6=94=BF?= =?UTF-8?q?=E5=BA=9C=E6=8C=87=E5=AF=BC=E4=BB=B7=E7=9A=84=E9=82=A3=E6=89=B9?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 15 ++++++++++-- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 24 ++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 9515d1d1b..845b06548 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -146,7 +146,9 @@ class CcbCommonUtils { public static def findGuidePrice(city, startDate, endDate){ def q = new BasicDBObject() q.put("city", city) - q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); + if(startDate && endDate){ + q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); + } def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q) def rets = [] for(def d : dbcursor){ @@ -428,7 +430,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","鸣翠蓝湾") +// q.put("name","锦南浦三小区") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) @@ -660,4 +662,13 @@ class CcbCommonUtils { pgsqlInstance.executeUpdate(sql2) } + + public static def findDistGovPrice(def city){ + def ids = [] + pgsqlInstance.eachRow("SELECT gujia_bds_id from base.t0831_community_guide_price_gj_h where urbn_nm='${city}'",{it-> + ids << it.gujia_bds_id + }); + return ids; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 7f6dd3e59..84bbeaa2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -46,6 +46,15 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { + +// def gocPrices = CcbCommonUtils.findDistGovPrice("深圳") +// def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val=202303 and gujia_bds_id=?" +// pgsqlInstance.withBatch(500, sql){ ps-> +// for(def id : gocPrices){ +// ps.addBatch(id) +// } +// } + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 @@ -53,13 +62,20 @@ class ThirdPartyPriceBatch extends BaseBatch { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ + + def gocIdsPrices = [] + if(city == "深圳"){ + gocIdsPrices = CcbCommonUtils.findDistGovPrice(city) + } + def statMap = getDistrictStatByCity(city, weekend) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) def thirdPriceMap = findThirdPartyPrice(city, statMap) //def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用) - def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap) + + def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices) // def rets = createData3(cityCodeDatas.get(city))//新第三方均价表,要做2步,麻烦 rows.addAll(rets) println index+" "+city+" "+rets.size() +" "+rows.size() @@ -187,10 +203,12 @@ class ThirdPartyPriceBatch extends BaseBatch { } - def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){ def allRows = [] for(def d : dists){ - + if(gocPrices && gocPrices.contains(d._id.toString())){ + continue; + } def third = thirdPriceMap.get(d.region+"##"+d.name) if(third){ def data = distCodeMap.get(d.region) From 002257d674c3153a37f3b67e8690bbfefdd2bd57 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Apr 2023 16:36:27 +0800 Subject: [PATCH 12/19] update --- .../BigdataCommunityBatch.groovy | 80 +++++++++++++++++-- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index cd0d1a45b..4edbf78a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 -// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 +// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -78,9 +78,13 @@ class BigdataCommunityBatch extends BaseBatch{ println "txt 生成->导Oracle" -// delPgData() -// saveToPgSql(rows) + delPgData() + saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") + +// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") +// delPgData() +// saveToPgSql2(datas) } def addDeleteDist(dists, ccbMaps, rets){ @@ -255,14 +259,74 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - sqlInstance.withBatch(20000, sql){ ps-> + sqlInstance.withBatch(1, sql){ ps-> for(def r : rows) { - ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], - r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) - ps.toString() + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + }catch(Exception e){ + e.printStackTrace() + } } } println "end: "+new Date() } + + def saveToPgSql2(rows){ + def newRows = [] + for(def r : rows) { + def newR = [] + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ +// r[i] = null + newR << null + }else{ + newR << r[i] + } + } + newR << null + newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + newR << null + newR << '2999-12-31' + newR << null + newR << null + newR << null + + newRows << newR + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + def index = 0 + sqlInstance.withBatch(10000, sql){ ps-> + for(def r : newRows) { + index++ + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + index + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("\\|@\\|") + rows << row + } + return rows; + } + } From 343c3a0cec6544e893172f7d826ff96f32a0fac0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 21 Apr 2023 14:10:42 +0800 Subject: [PATCH 13/19] update --- .../ccbevaluation/BigdataDealBatch.groovy | 84 +++++++++---------- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +- .../ronghe/ExportDistrictRongHe.groovy | 23 ++--- 3 files changed, 57 insertions(+), 54 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 5e7d45516..e00fdeb41 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -67,50 +67,50 @@ class BigdataDealBatch extends BaseBatch{ } println "curDate:"+curDate - for(def city : citys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } - } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 // for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } // } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -// saveToPgSql(rows) + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt 生成 " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 4ec047367..716114623 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-02-27", "2023-03-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-21") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) //更新静态数据表 -// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index e25da703d..ce4355ead 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -66,10 +66,7 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) - //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 - def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); -// exportThirdPartyPrice2(thirdPartyMonth, new_dists) - exportPgThirdPartyPrice2(thirdPartyMonth, new_dists) + //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 // exportCommunityBasePrice(lastMonth) @@ -78,16 +75,22 @@ class ExportDistrictRongHe extends BaseBatch{ // exportBigdataSupplierStatistics(lastMonth) // exportBigdataModelingBase(lastMonth) - exportPgCommunityBasePrice(lastMonth) - exportPgCommunityPdescribeGj(lastMonth) - exportPgGuideStatistics(lastMonth) - exportPgBigdataSupplierStatistics(lastMonth) - exportPgBigdataModelingBase(lastMonth) - //2月8号 临时导出苏州一年的数据 exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 } + if(curDate.toString().endsWith("28")){ + exportPgBigdataSupplierStatistics(lastMonth) + //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); +// exportThirdPartyPrice2(thirdPartyMonth, new_dists) + exportPgThirdPartyPrice2(thirdPartyMonth, new_dists) + exportPgCommunityBasePrice(lastMonth) + exportPgCommunityPdescribeGj(lastMonth) + exportPgBigdataModelingBase(lastMonth) + exportPgGuideStatistics(lastMonth) + } + SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); From ae87f3f055733a894dd5c03b9266cc15e1cc480a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 21 Apr 2023 15:04:21 +0800 Subject: [PATCH 14/19] =?UTF-8?q?=E5=86=99=E5=AD=97=E6=A5=BC=E5=8C=BA?= =?UTF-8?q?=E5=9F=9F=E5=9B=B4=E6=A0=8F=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/crawl/OfficeProcess.groovy | 515 ++++++++++++++++++ 1 file changed, 515 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy new file mode 100644 index 000000000..0a57c35c6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy @@ -0,0 +1,515 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import groovy.sql.Sql + +import java.sql.Connection +import java.sql.DriverManager +import java.sql.PreparedStatement +import java.sql.Statement +import java.util.regex.Pattern + +class OfficeProcess { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + def sqlInstance = Sql.newInstance("jdbc:clickhouse://192.168.0.216:8123/etl?max_memory_usage=50000000", + "goojia", "goojia123456", "ru.yandex.clickhouse.ClickHouseDriver") + + static main(args) { + def filePath = "D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx" + def task = new OfficeProcess(); + +// task.run("D:\\数据\\写字楼\\6城合并写字楼.xlsx") +// task.addPropertyType2("深圳") + +// task.standardName(filePath) +// task.nameRepeatFiter(filePath) +// task.addressFiter(filePath) +// task.nameFiter(filePath) +// task.typeFiter(filePath) +// task.nameFlag(filePath) +//// task.updateFlagToDB(filePath) +// task.updateFlagToDB2(filePath) + + task.checkRegion() + + } + + def nameRepeatFiter(filePath){ + def keys = [] + ExcelUtils.modify(filePath, { row -> + if (!row['L'] && row['B']) { + def city = row['B'] + def region = row['C'] + def name = "" + if(row['K']){ + name = row['K'] + }else{ + name = row['D'] + } + def key = city+"##"+region+"##"+name + if(keys.contains(key)){ + row['L'] = "名称去重" + println key + }else{ + keys << key + } + } + }) + } + + def nameFiter(filePath){ + ExcelUtils.modify(filePath, { row -> + def name = row['K'] + + if(!row['L'] && !name){ + row['L'] = "名称过滤" + } + if(!row['L'] && name){ + if(name.contains("公司")){ + row['L'] = "名称过滤" + } + } + + if(!row['L'] && name){ + if(isStandardRoad(name)){ + row['L'] = "名称过滤" + } + } + + + + if(!row['L'] && name){ + def fiters = ["厂房","测试", "服务","综合服务","酒店","深圳特专","工业","经济","产业","居委会","宿舍","购物","休闲","文化","训练","出租","浙菜","值班岗","招租","商住","办公室","配套","美食","博物馆","物流","商铺","公社","商城","值班室","暂停","创业社区","招商","宾馆","器材","教育","门诊","妇产科","商贸城","电子城","超市","零售"] + for(def key : fiters){ + if(name.contains(key)){ + row['L'] = "名称过滤" + break; + } + } + } + + if(!row['L'] && name){ + def fiters = ["大厦","裙楼",'花园','大楼','大堂','南塔','北京','大道','主楼','中国','中心','广场','鸟巢','小区','单位','办公','北塔' + ,'写字楼','检验楼','办公楼','接待区','商务楼','文体楼','市中心','产业园','科技楼','行政楼','病房43、44、45','律己楼','调度楼','辉腾楼','总务楼' + ,'7幢货梯','现厂B栋','(马连道南街)','中央电视台','后座','西大楼','B&C','商业用楼','财务中心','地铁','一瓶','外贸楼','业务楼','研发大楼','结算中心','马场','商务'] + for(def key : fiters){ + if(name == key){ + row['L'] = "名称过滤" + break; + } + } + } + + + if(!row['L'] && row['J'] && name){ + if(row['J'].toString().contains("商务写字楼") && name.contains("公寓")){ + row['L'] = "名称过滤" + } + } + + if(!row['L'] && name){ + if(findPattern(name,'^[A-Za-z0-9]+$') && name.toString().length()<=3){ + row['L'] = "名称过滤" + }else if(name.toString().length()<2){ + row['L'] = "名称过滤" + } + } + + if(!row['L'] && name){ + def filters = ['筹备组','小区','家属房','家属院','家属楼','自编','市场','项目','批发城','传达室','梯','型材','学院','工地','冷冻厂','机关楼','阅读室','议事厅','电脑城','童装城','派出所','保安室','散盘','店','工作站',"物业楼",'物业','驿站','锅炉房','处理中心',"销售中心",'政府','馆','捐赠点','百货','工厂','工场','检测场','职工之家','移动','中心站','楼门','职场','鲜花基地','医院','装饰城','实训楼','业务楼','科研楼','学校','交易城','管理楼','考场','通道','团购','楼上','运营中心','商业城'] + for(def key : filters){ + if(name.toString().endsWith(key)){ + row['L'] = "名称过滤" + break; + } + } + } + }) + } + + def typeFiter(filePath){ + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['J']){ + def fiters = ["商住","服务","住宅小区","酒店","商业综合体","住宅底商"] + for(def key : fiters){ + if(row['J'].toString().contains(key)){ + row['L'] = "类型过滤" + break; + } + } + } + if(!row['L'] && row['B'] && !row['J']){ + row['L'] = "类型空白过滤" + } + }) + } + + def nameFlag(filePath){ + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['D']) { + def fiters = ["大厦","大楼","商务楼","商务中心","广场","写字楼","办公楼","科技园","国际大厦","商务大厦","商务广场","创园","创意","中心","营销中心"] + for(def key : fiters){ + if(row['D'].toString().contains(key)){ + if(row['D']=="广场"){ + + } + row['L'] = "名称关键字[添加]" + break; + } + } + } + + if(!row['L'] && row['J']){ + if(row['J'].toString().contains("园区") || row['J'].toString().trim()=="纯写字楼" || row['J'].toString().trim()=="标准写字楼"){ + row['L'] = "类型[添加]" + } + + if(!row['L'] && row['J'].toString().trim()=="纯写字楼 标准写字楼" || row['J'].toString().trim()=="商务中心" || row['J'].toString().trim()=="商务中心 标准写字楼"){ + row['L'] = "类型[添加]" + } + } + + if(!row['L'] && row['J'] && row['D']){ + if(row['J'].toString().contains("商务写字楼") && !row['D'].toString().contains("公寓")){ + row['L'] = "类型[添加]" + } + } + + if(!row['L'] && row['J'] && row['D']){ + if(row['J'].toString().contains("纯写字楼")){ + row['L'] = "类型[添加]" + } + } + + }) + + } + + def addressFiter(filePath){ + def keyMap = [:] + ExcelUtils.modify(filePath, { row -> + if(!row['L'] && row['F']){ + def city = row['B'] + def region = row['C'] + def name = "" + if(row['K']){ + name = row['K'] + }else{ + name = row['D'] + } + def address = row['F'] + def key = city+"##"+region+"##"+address + def keys = keyMap.keySet() + if(keys.contains(key)){ + def values = keyMap.get(key) + if(values){ + for(def n : values){ + def score = CosineSimilarAlgorithm.getSimilarityRatio(name, n) + if(score>=0.8){ + row['L'] = "地址去重" + break; + } + println address +" "+ name+" "+n +" "+score + } + } + + }else{ + def values = keyMap.get(key) + if(!values){ + values = [] + keyMap.put(key, values) + } + values << name + } + } + + }); + } + + def standardName(filePath){ + def index = 0 + ExcelUtils.modify(filePath, { row -> + index++ + if(!row['K'] && row['D']){ + def name = row['D'] + def newName = name.replaceAll("-|·|(|)|//|\\?|“|”","").replaceAll("\\w+幢\\w+座","").replaceAll(" ","").replaceAll("\\.","").replaceAll("丨","") + newName = newName.replaceAll("-?\\w+号商业楼","").replaceAll("\\?","") + newName = newName.replaceAll("\\w+楼\\w+室","") + newName = newName.replaceAll("\\w+号\\w+(座|幢)","") + newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十)号楼(一|二|三|四|五|六|七|八|九|十)单元","") + newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|十一)?幢?(一|二|三|四|五|六|七|八|九|十|东|南|西|北)(塔楼|楼)","") + newName = newName.replaceAll("-?(第)?\\w+(号写字楼|号楼|号大楼|楼|座|栋|南楼|北楼|东楼|西楼|区|单元)","") + newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾)栋","") + newName = newName.replaceAll('(东|南|西|北)座$',"") + newName = newName.replaceAll("1期","一期").replaceAll("2期","二期").replaceAll("3期","三期").replaceAll("4期","四期").replaceAll("5期","五期").replaceAll("6期","六期").replaceAll("7期","七期").replaceAll("8期","八期").replaceAll("9期","九期").replaceAll("10期","十期").replaceAll("1-5期","一-五期").replaceAll("5-6期","六-七期") + newName = newName.toUpperCase().replaceAll("(装修中)","").replaceAll("\\(\\)","") + + if(newName){ + def md = DataUtil.match(newName, Pattern.compile("^[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中]{0,1}[写字|办公|楼|座|区|栋|组|幢|0-9|东|南|西|北|中|号]?[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中|办公|写字]?[楼|座|区|栋|组|幢|0-9|东|南|西|北|中]\$")) + if(md){ + newName = '' + } + } + row['K'] = newName + } + }) + } + + def addPropertyType2(runCity){ + def map = [:] + def datas = ExcelUtils.read("D:\\数据\\写字楼\\6城未匹配写字楼.xlsx",0,1) + if(datas){ + for(def d : datas){ + def key = d['B']+"##"+d['C']+"##"+d['D'] + def list = map.get(key) + if(!list){ + list = [] + map.put(key, list) + } + if(d['M'] && !list.contains(d['M'])){ + list << d['M'] + } + if(d['U'] && !list.contains(d['U'])){ + list << d['U'] + } + + } + } + + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx", { row -> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def types = map.get(city+"##"+region+"##"+name) + if(types){ + row['J'] = types.join(" ") + } + }) + } + + def addPropertyType(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\写字楼\\未匹配写字楼.xlsx", { row -> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def types = findDatasByDB(city, region, name) + if(types){ + row['J'] = types.join(" ") + } + println index + }) + } + + def findDatasByDB(city, region, name){ + def sql = "select * from office where city='${city}' and region='${region}' and name='${name}'" + def types = [] + sqlInstance.eachRow(sql, {r-> + if(r.property_type && !types.contains(r.property_type)){ + types << r.property_type + } + }) + return types + } + + def run(filepath){ + def index = 0 + ExcelUtils.multiThreadingModify(filepath,{row-> + index++ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def address = row['I'] + if(city ){ + def jsonObject = OpenApiClient.addressMatch(city, name, "office", false, region, null) +// if(jsonObject && jsonObject.precise == -1){ +// jsonObject = OpenApiClient.addressMatch(city, address, "office", false, region, name) +// } + if (jsonObject != null && jsonObject.precise != -1) { + row['J'] = jsonObject.get("id") + row['K'] = jsonObject.get("city") + row['L'] = jsonObject.get("district") + row['M'] = jsonObject.get("name") + row['N'] = jsonObject.get("credit") + row['O'] = jsonObject.get("propertyType") + } + } + println index + }) + } + + def isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)?"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + if(s.toString().endsWith("路")||s.toString().endsWith("街")){ + return true; + } + return false + } + + + def findPattern(def address, def regex) { + Pattern pattern = Pattern.compile(regex); + def matcher = pattern.matcher(address) + if (matcher.find()) { + return true + } + return false; + } + + def updateFlagToDB(filePath){ + def datas = ExcelUtils.read(filePath,0,1) + def total = datas.size() + def index = 0 + for(def d : datas){ + def sql = "alter table office update option='${d['L']}', format_name='${d['K']}' where city='${d['B']}' and region='${d['C']}' and name='${d['D']}';" + index++ + sqlInstance.executeUpdate(sql) + println index+"/"+ total + } + } + + + def updateOption(city, region, name, source, flag){ + def sql = "alter table office update region_fence='${flag}' where city='${city}' and region='${region}' and name='${name}' and source='${source}';" + sqlInstance.executeUpdate(sql) + } + + def batchUpdate(rets){ + // 初始化驱动 + Class.forName("ru.yandex.clickhouse.ClickHouseDriver"); + // url + String url = "jdbc:clickhouse://192.168.0.216:8123/etl"; + // 用户名密码 + String user = "goojia"; + String password = "goojia123456"; + // 登录 + Connection connection = DriverManager.getConnection(url, user, password); + Statement stmt = connection.createStatement(); + def count = 0 + def sql = "alter table office update region_fence=? where city=? and region=? and name=? and source=?;" + PreparedStatement pstmt = connection.prepareStatement(sql) + + for(def d : rets){ + count++ + pstmt.setString(1, d.flag) + pstmt.setString(2, d.city) + pstmt.setString(3, d.region) + pstmt.setString(4, d.name) + pstmt.setString(5, d.source) + println count+" "+d.city+" "+d.name + pstmt.addBatch() + if(count % 1 ==0) { + try { + pstmt.executeBatch() + } catch (Exception e) { + e.printStackTrace() + } + } + } + pstmt.executeBatch() + } + def updateFlagToDB2(filePath){ + // 初始化驱动 + Class.forName("ru.yandex.clickhouse.ClickHouseDriver"); + // url + String url = "jdbc:clickhouse://192.168.0.216:8123/etl"; + // 用户名密码 + String user = "goojia"; + String password = "goojia123456"; + // 登录 + Connection connection = DriverManager.getConnection(url, user, password); + Statement stmt = connection.createStatement(); + def count = 0 + def datas = ExcelUtils.read(filePath,0,1) + def sql = "alter table office update option=?, format_name=? where city=? and region=? and name=?;" + PreparedStatement pstmt = connection.prepareStatement(sql) + + for(def d : datas){ + if(d['B']){ + count++ + pstmt.setString(1, d['L']) + pstmt.setString(2, d['K']) + pstmt.setString(3, d['B']) + pstmt.setString(4, d['C']) + pstmt.setString(5, d['D']) + println count+" "+d['L']+" "+d['K']+" "+d['B']+" "+d['C']+" "+d['D'] + pstmt.addBatch() + if(count % 100 ==0) { + try { + pstmt.executeBatch() + } catch (Exception e) { + + } + } + } + } + pstmt.executeBatch() + } + + def checkRegion(){ + def citys = ["深圳","北京","广州","重庆","天津","成都","南京"]//"上海", + for(def city : citys){ + def regionPointsMap = findRegionRange(city) + def addDatas = findAddDatasByClickhouse(city) + println "需处理的区域围栏:"+addDatas.size() + def index = 0 + def rets = [] + for(def data : addDatas){ + index ++ + if(data.lng){ + def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region)) + def mark = flag?'T':'F' +// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark] + updateOption(data.city, data.region, data.name, data.source, mark) + println index+" "+data.region + " "+data.name+" "+ flag + } + } +// batchUpdate(rets) + } + } + + def findAddDatasByClickhouse(city){ + def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%添加%'" + def datas = [] + sqlInstance.eachRow(sql, {r-> + datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source] + }) + return datas + } + + def findRegionRange(city){ + def q = new BasicDBObject() + q.put("city",city) + def dbcursor = POLYLINECOLL.find(q) + def regionMap = [:] + for(def d : dbcursor){ + def region = RegionData.formatRegion(city,d.region) + regionMap.put(region, d.polyline) + } + return regionMap + } + + def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; + } +} From 22c11ea8cd587b7e142d44c93dd941e23fb28497 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 21 Apr 2023 15:24:50 +0800 Subject: [PATCH 15/19] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy index 0a57c35c6..ba3adf88d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy @@ -465,7 +465,7 @@ class OfficeProcess { } def checkRegion(){ - def citys = ["深圳","北京","广州","重庆","天津","成都","南京"]//"上海", + def citys = ["北京","广州","重庆","天津","成都","南京"]//"上海","深圳", for(def city : citys){ def regionPointsMap = findRegionRange(city) def addDatas = findAddDatasByClickhouse(city) From 832450a795e1d3717615ea80345fdd2e4fd4fb2f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 21 Apr 2023 16:07:17 +0800 Subject: [PATCH 16/19] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/crawl/OfficeProcess.groovy | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy index ba3adf88d..e422b860f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.crawl import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point import com.fangjia.script.util.CosineSimilarAlgorithm @@ -26,7 +27,11 @@ class OfficeProcess { "goojia", "goojia123456", "ru.yandex.clickhouse.ClickHouseDriver") static main(args) { - def filePath = "D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx" + def filePath = "/home/goojia/gj/6城未匹配上的写字楼.xlsx" + + if(CommonDatautils.isWindow()){ + filePath = "D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx" + } def task = new OfficeProcess(); // task.run("D:\\数据\\写字楼\\6城合并写字楼.xlsx") @@ -38,10 +43,10 @@ class OfficeProcess { // task.nameFiter(filePath) // task.typeFiter(filePath) // task.nameFlag(filePath) -//// task.updateFlagToDB(filePath) + task.updateFlagToDB(filePath) // task.updateFlagToDB2(filePath) - task.checkRegion() +// task.checkRegion() } @@ -452,12 +457,8 @@ class OfficeProcess { pstmt.setString(5, d['D']) println count+" "+d['L']+" "+d['K']+" "+d['B']+" "+d['C']+" "+d['D'] pstmt.addBatch() - if(count % 100 ==0) { - try { - pstmt.executeBatch() - } catch (Exception e) { - - } + if(count % 10 ==0) { + pstmt.executeBatch() } } } From d540103feb027cfaee61cdeb6e5dac4e2b20b7ff Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 22 Apr 2023 16:27:24 +0800 Subject: [PATCH 17/19] updadte --- .../script/gj/crawl/OfficeProcess.groovy | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy index e422b860f..231a930c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/OfficeProcess.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.crawl import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point import com.fangjia.script.util.CosineSimilarAlgorithm @@ -43,10 +44,10 @@ class OfficeProcess { // task.nameFiter(filePath) // task.typeFiter(filePath) // task.nameFlag(filePath) - task.updateFlagToDB(filePath) +// task.updateFlagToDB(filePath) // task.updateFlagToDB2(filePath) -// task.checkRegion() + task.checkRegion() } @@ -382,13 +383,26 @@ class OfficeProcess { def updateFlagToDB(filePath){ def datas = ExcelUtils.read(filePath,0,1) def total = datas.size() + println total def index = 0 + def rets = [] for(def d : datas){ def sql = "alter table office update option='${d['L']}', format_name='${d['K']}' where city='${d['B']}' and region='${d['C']}' and name='${d['D']}';" index++ - sqlInstance.executeUpdate(sql) - println index+"/"+ total + rets << sql + //sqlInstance.executeUpdate(sql) + //println index+"/"+ total + println sql } + + + def filePath1 = "d:\\Data1.txt" + File file = new File(filePath1) + def s = new StringBuffer() + for(def r : rets){ + s.append(r+"\n") + } + file.write(s.toString()) } @@ -457,7 +471,7 @@ class OfficeProcess { pstmt.setString(5, d['D']) println count+" "+d['L']+" "+d['K']+" "+d['B']+" "+d['C']+" "+d['D'] pstmt.addBatch() - if(count % 10 ==0) { + if(count % 100 ==0) { pstmt.executeBatch() } } From ad046713b50b188d4543ab80d1752f2457ec0378 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 26 Apr 2023 18:18:40 +0800 Subject: [PATCH 18/19] =?UTF-8?q?=E5=85=88=E5=8E=BB=E6=8E=89=E6=9D=83?= =?UTF-8?q?=E9=99=90=E9=AA=8C=E8=AF=81=EF=BC=8C=E6=B1=9F=E5=8D=97=E8=A1=8C?= =?UTF-8?q?=E6=88=BF=E6=BA=90=E6=8E=A5=E5=8F=A3=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/HouseController.java | 4 +- .../open/controllers/JnBankController.java | 2 +- .../datastore/CzbankSampleServiceImpl.java | 115 ++++++++++++++++-- 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index 1efa12d0f..5f2fddf88 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -208,7 +208,7 @@ public class HouseController { } - @Authority(role = Role.USER) + @Authority(role = Role.NONE) public Object similarAction(@Valid CollateralSampleQuery collateralSampleQuery){ RestfulModelAndView mav = new RestfulModelAndView(); @@ -253,7 +253,7 @@ public class HouseController { /**商铺相似房源接口*/ - @Authority(role = Role.USER) + @Authority(role = Role.NONE) public Object shopSimilarAction(@Valid ShopCollateralSampleQuery query){ RestfulModelAndView mav = new RestfulModelAndView(); ShopCollateralSampleQueryBean queryBean = new ShopCollateralSampleQueryBean(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java index 2c7a22187..998b4664e 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/JnBankController.java @@ -39,7 +39,7 @@ public class JnBankController { private static final Pattern STAGE_PATTERN = Pattern.compile(".*[\\w东南北中西甲乙丙丁一二三四五六七八九十百千\\d]+(期|区)$"); - @Authority(role = Role.USER) + @Authority(role = Role.NONE) public Object searchAction(@Valid JNBankSearchModel model){ RestfulModelAndView mav = new RestfulModelAndView(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java index 604d35021..52cea9435 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/CzbankSampleServiceImpl.java @@ -1,9 +1,6 @@ package com.fangjia.service.datastore; -import com.fangjia.core.util.BeanUtils; -import com.fangjia.core.util.DataUtil; -import com.fangjia.core.util.DateUtil; -import com.fangjia.core.util.HouseValidator; +import com.fangjia.core.util.*; import com.fangjia.dal.QueryParam; import com.fangjia.dal.SortParam; import com.fangjia.dal.SrQuery; @@ -51,7 +48,7 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ private static String[] HOUSE_FIELDS = new String[]{"city", "region", "districtName", "totalPrice", "avgPrice", "area", "fitment", "direction", "floor", "height", "publishDate", "address", "estateType"}; - + public static String[] TAGS =new String[]{"独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"}; private DealPriceDao dealPriceDao; private HouseDao houseDao; @@ -191,7 +188,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ } List querySamples= new ArrayList(); List samples = findSampleByDistrict(district.getCity(), district.getRegion(), district.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat); - scoreForSample(samples, target); +// scoreForSample(samples, target); + sortForSample(samples, target); for(SampleWrapper sample : samples){ sample.setProperty(queryBean.getProperty()); } @@ -205,7 +203,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ for(SampleWrapper sample : nearlySample){ sample.setProperty(d); } - scoreForSample(nearlySample, target); +// scoreForSample(nearlySample, target); + sortForSample(samples, target); querySamples.addAll(nearlySample); if(querySamples.size() > max){ break; @@ -481,6 +480,56 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ return samples; } + + private List sortForSample(List samples, final SampleWrapper targetSample){ + + if(samples.isEmpty()){ + return samples; + } +// if(samples!=null && samples.size() < max){ +// return samples; +// } + Collections.sort(samples, new Comparator() { + public int compare(SampleWrapper o1, SampleWrapper o2) { + Double d1 = Math.abs(o1.getStrctureArea() - targetSample.getStrctureArea()); + Double d2 = Math.abs(o2.getStrctureArea() - targetSample.getStrctureArea()); +// return (int) (d1 - d2); + return d1.compareTo(d2); + } + }); + + List prices = new ArrayList(); + for(SampleWrapper sample : samples){ + if(!prices.contains(sample.getUnitPrice())){ + prices.add(sample.getUnitPrice()); + if(prices.size()>=10){ + break; + } + } + } + final Integer midPrice = (int)median(prices); + Collections.sort(samples, new Comparator() { + public int compare(SampleWrapper o1, SampleWrapper o2) { + Integer d1 = Math.abs(o1.getUnitPrice() - midPrice); + Integer d2 = Math.abs(o2.getUnitPrice() - midPrice); + return d1.compareTo(d2); + } + }); + return samples; + } + + private static double median(List total) { + double j = 0; + Collections.sort(total); + int size = total.size(); + if(size % 2 == 1){ + j = total.get((size-1)/2); + }else { + j = (total.get(size/2-1) + total.get(size/2) + 0.0)/2; + } + return j; + } + /** * 查找小区样本 * @param city @@ -514,8 +563,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ houseQuery.add(new QueryParam("region", region )); if(districtStat != null && districtStat.getAvgPrice() > 0){ int avgPrice = districtStat.getAvgPrice(); - houseQuery.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_EQUAL_OPERATOR, (int)(avgPrice * 0.85 ))); - houseQuery.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_EQUAL_OPERATOR, (int)(avgPrice * 1.15 ))); + houseQuery.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_EQUAL_OPERATOR, (int)(avgPrice * 0.5 )));//0.85 + houseQuery.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_EQUAL_OPERATOR, (int)(avgPrice * 1.5 )));//1.15 } BooleanQuery nameQuery = new BooleanQuery(); int c = 0; @@ -548,6 +597,21 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ int maxFloor = 0; boolean allSampleHasHeight = true; for(House house : houses){ + if(StringUtils.isNotBlank(estateType) && estateType.equals("01")){//01:普通住宅 02:别墅 + if(StringUtils.isNotBlank(house.getEstateType()) && house.getEstateType().contains("别墅")){ + continue; + } + if(isVilla(house.getArea(), house.getName(), house.getHeight(), house.getRoom())){ + continue; + } + }else if(StringUtils.isNotBlank(estateType) && estateType.equals("02")){ + if(house.getHeight()!=null && house.getHeight()>=10){ + continue; + } + if(house.getFloor()!=null && house.getFloor()>=10){ + continue; + } + } SampleWrapper sample = convertHouse2Sample(house); if(isAvailableSample(sample)){ @@ -859,6 +923,39 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{ return null; } + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(Double area, String name, Integer height, Integer room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + public void setDealPriceDao(DealPriceDao dealPriceDao) { this.dealPriceDao = dealPriceDao; } From d1c1a4d934a6416455da6a909533f165f03f325a Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 27 Apr 2023 09:57:45 +0800 Subject: [PATCH 19/19] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8B=9B=E5=95=86?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index f998411a0..ba0985a47 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1861,6 +1861,44 @@ public class AddressController { return mav; } + /** + * 招商证券POC测试接口封装 + */ + public Object cmbTestAction(AddesssMatchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/cmbTest", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/cmbTest", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + /** * 五级地址标准化接口 *