From 3f35d0cb71fa046e4f20aa312351b7fc3946e0c6 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 30 May 2023 15:33:08 +0800 Subject: [PATCH 01/22] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B1=85=E5=A7=94?= =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/CommitteeLatLng.groovy | 4 ++-- .../temp/shenjian/util/GaoDeUtils.groovy | 22 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index c1a7d36eb..db1f5a13d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -24,7 +24,7 @@ class CommitteeLatLng { static ExecutorService executorService = Executors.newFixedThreadPool(10) static void main(String[] args) { - dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021)) + dealPoi(new BasicDBObject("location", "").append("year", 2022)) //dealLocation() executorService.shutdown() } @@ -71,7 +71,7 @@ class CommitteeLatLng { if (amapPoi && amapPoi.size() > 0) { def poi = amapPoi[0] double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, name) - if (similarity <= 0.8) { + if (similarity <= 0.8 && !String.valueOf(poi.name).contains(name)) { amapPoi = GaoDeUtils.searchPoi(city, address, area, "") } poi = amapPoi[0] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 2a34dd4b3..30b2e3a65 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -55,13 +55,17 @@ class GaoDeUtils { * @return */ static String getGaodeKey(String type) { - Map params = new HashMap<>(); - params.put("type", type) - JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params) - if (json.status == 0 && json.result != null) { - return json.result - } - System.exit(1); +// Map params = new HashMap<>(); +// params.put("type", type) +// JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params) +// if (json.status == 0 && json.result != null) { +// return json.result +// } +// System.exit(1); + return "8796b9dfb3889b57c38e2fff1b7e51bd"; +// 'eb69e4ad2a9cdd31f163a0ed32effe6e', '60f764d228d64addf390e28d68a01a35', '66e6771767ab293ace441553c36621c1', +// 'a63254b4a1b2e6f4bb11058e0b5d9fa2', '743ad872ad39f0a10665b28fa968af0b', '3b7c1d201c403bb416b63535ed3d0e98', +// 'a34894d151fffff94f9a33bae3d5565c', '389e0a3a155e8d28728c3faede4ebd4f' } /** @@ -330,10 +334,10 @@ class GaoDeUtils { params.page = "${page}".toString() page++ params.key = this.getGaodeKey("v5/place/text") - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); if (json == null) { params.key = this.getGaodeKey("v5/place/text") - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() From 3da40ce36f9a5121ac857441f604572376f105ba Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 20 Jun 2023 11:22:46 +0800 Subject: [PATCH 02/22] update --- .../script/ccbevaluation/AddSzDealData.groovy | 24 ++++++- .../BigdataCommunityBatch.groovy | 9 ++- .../ronghe/BigdataCitypriceBatch.groovy | 6 +- .../script/chongqing/HailanTest.groovy | 66 ++++++++++++++++- .../script/commData/CommonDatautils.groovy | 71 ++++++++++++++++++- .../script/fuzhou/FzDataResultStat.groovy | 2 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../district/AddDistrictElevatorBatch.groovy | 2 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 28 +++++--- .../gj/paimai/LianjiaCrawlByChrome.groovy | 48 ++++++++----- .../DayUpdateDistRefreshBatch.groovy | 15 +++- .../script/jiangnanblank/FzDataUtils.groovy | 7 ++ .../thirdPriceCrawl/DataStandardBatch.groovy | 11 +-- .../ThirdDataToGatherBatch.groovy | 2 +- 15 files changed, 241 insertions(+), 56 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index f34d40421..1c1fa2788 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -51,7 +51,19 @@ for(def d : datas){ map.region = d['B'] def roomStr = d['E'] if(roomStr){ - map.room = roomStr.toString().toDouble().toInteger() + if(roomStr.contains("-")){ + def arrs = roomStr.split("-") + try{ + map.room = arrs[0].toString().toInteger() + }catch(Exception e){ + println "room error" + } + map.hall = arrs[1].toString().toInteger() + map.toilet = arrs[3].toString().toInteger() + }else{ + map.room = roomStr.toString().toDouble().toInteger() + } + } map.area = d['F'].toString().toDouble() @@ -59,7 +71,13 @@ for(def d : datas){ map.avg_price = (int)(d['H'].toString().toDouble()) map.sell_price = (int)(d['I'].toString().toDouble()*10000) def trade_date = d['J'] - map.trade_date = trade_date + if(trade_date){ + map.trade_date = trade_date + }else{ + def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31") + map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") + } + map.source = "深房家族" map.create_date = new Date() map.m_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 17860e977..42feb5869 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 -// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 +// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -77,9 +77,8 @@ class BigdataCommunityBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成->导Oracle" - -// delPgData() -// saveToPgSql(rows) + delPgData() + saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index ed89f59ac..9ed8300cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -61,9 +61,9 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); - cityRun(citys, startDate, endDate, cityCodeMap) - regionRun(citys, startDate, endDate, cityCodeMap) -// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) +// cityRun(citys, startDate, endDate, cityCodeMap) +// regionRun(citys, startDate, endDate, cityCodeMap) + districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } def cityRun(citys, startDate, endDate, cityCodeDataMap){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy index 0d0e263e6..ff1b235f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -40,19 +40,25 @@ class HailanTest { static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new HailanTest(); task.run(); } def run(){ - def inputfile = "D:\\数据\\微元素\\排查表.xlsx" - match(inputfile) + def inputfile = "D:\\数据\\微元素\\test.xlsx" +// addThirdPrice(inputfile) +// match(inputfile) +// tradeAvgPrice(inputfile) +// caluateTotalPrice(inputfile) // addPrice(inputfile) // addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx") // addHouse(inputfile) // stat(inputfile) // hailanStat(inputfile) + addFzFjwPrice(inputfile) } def addHouse(inputfile){ @@ -702,4 +708,60 @@ class HailanTest { } return prices; } + + def tradeAvgPrice(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){ + def dist = CommonDatautils.findDistrictById(row['I']) + if(dist){ + def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01"); + row['J'] = (int)tradeAvgPrice + } + } + }) + } + + + def addThirdPrice(path){ + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 + + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate) + def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95) + } + } + }) + } + + def caluateTotalPrice(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['J']){ + row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble()) + }else if(row['K']){ + row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble()) + } + }) + } + + def addFzFjwPrice(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['B']) { + def q = new BasicDBObject() + q.put("city", "福州") + q.put("region", row['A']) + q.put("name", row['B']) + q.put("month", 202304) + def obj = FZ_DISTRICT_STAT_COLL.findOne(q) + if (obj) { + row['D'] = obj.avg_price + } + } + + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 2ec846add..bc5abebe7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1022,7 +1022,7 @@ class CommonDatautils { q.put("cat", cat); q.put("region", region) q.put("name", name); - q.put("status", 0) + q.put("visible", 0) DBObject keys = new BasicDBObject(); if(fields){ @@ -3801,5 +3801,74 @@ class CommonDatautils { } + static def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + + static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.beike_esf_price) + } + } + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 2da506d11..917ddfc5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -25,7 +25,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); -// task.regionPriceTrendRun() + task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index a3be166aa..63211976b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 16cae9383..4de90b96b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-04-01" + static def statDate = "2023-05-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy index 10efae15e..cc9c87682 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy @@ -65,7 +65,7 @@ class AddDistrictElevatorBatch extends BaseBatch{ } def newHouses = analyElevator(haveHouses, dist.estate_type2) if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){ - println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 10层以上:"+newHouses.size() + println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size() BasicDBObject udata = new BasicDBObject() if(dist.estate_type2 == "别墅"){ udata.put("is_elevator", false) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 1068d1f84..b190a324a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -289,20 +289,26 @@ public class BeiKeCrawlByChrome { } detail.source = "贝壳找房" - def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc) + def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc) + + def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","") if(liftStr){ detail.lift = DataFormatUtils.parseLift(liftStr) detail.lift_household = DataFormatUtils.parseHousehold(liftStr) } - detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") - detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) - detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) - detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) - detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) - detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) - detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) - detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) - def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡","")) + detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构","")) + detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡","")) + + detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","") + detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属","")) + detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途","")) + detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途","")) + detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限","")) if(detail.total_price && detail.avg_price){ detail.url = detailUrl @@ -483,7 +489,7 @@ public class BeiKeCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; - Thread.sleep(20000) + Thread.sleep(10000) }catch(Exception e){ Thread.sleep(15000) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0c82ca47e..9ab7eacb3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -297,16 +297,23 @@ public class LianjiaCrawlByChrome { def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") - if(dealDate.getTime()>=endDate.getTime()){ - detailUrls << durl - def dbo = findTrade(durl , "链家在线") - if (!dbo) { - parse(city, e, regionText, driver) + if(dealDate){ + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + println "1..."+new Date() + def dbo = findTrade(durl , "链家在线") + println "2..."+new Date() + if (!dbo) { + parse(city, e, regionText, driver) + } + }else{ + stop = true; + break; } }else{ - stop = true; - break; + println url +" 成交时间有误!" } + } if(page>=100){ stop = true; @@ -326,8 +333,9 @@ public class LianjiaCrawlByChrome { if(price){ def detailUrl = e.select(".title a").attr("href") driver.get(detailUrl) + println "3..."+new Date() def detailDoc = checkDocument(driver, detailUrl); - + println "4..."+new Date() def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -386,20 +394,22 @@ public class LianjiaCrawlByChrome { detail.source = "链家在线" detail.url = detailUrl - def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + def detailDiv = JQueryUtils.select(".introContent", detailDoc) + + def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","") if(liftStr){ detail.lift = DataFormatUtils.parseLift(liftStr) detail.lift_household = DataFormatUtils.parseHousehold(liftStr) } - detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") - detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) - detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) - detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) - detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) - detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) - detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) - detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) - def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层","")) + detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","") + detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构","")) + detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属","")) + detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限","")) + detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层","")) if(detail.total_price && detail.avg_price){ def houseInfo = e.select(".houseInfo").text() @@ -417,7 +427,9 @@ public class LianjiaCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail + println "5..."+new Date() DEAL_PRICE_COLL.save(detail) + println "6..."+new Date() } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy index bb19fe69d..72df7724e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy @@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{ if(dists){ for(def d : dists){ - def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) + def distObj = null + if(d.comm_id){ + distObj = CommonDatautils.findDistrictById(d.comm_id) + }else{ + distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) + } if(distObj && distObj._id){ - FzDataUtils.converDistrict(distObj, null, null); + if(d.type && d.type==1){ + FzDataUtils.removeById(d.comm_id) + }else{ + FzDataUtils.converDistrict(distObj, null, null); + } println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name updateTaskStatus(d.id, 2); } @@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{ def q = new BasicDBObject() q.put("status", 5);//未处理 TASK_COLL.find(q).each {d-> - rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name] + rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id] } return rets; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy index bf6e2c177..51e58e28e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy @@ -119,6 +119,13 @@ class FzDataUtils { } } + + static def removeById(id){ + def del_data = new BasicDBObject() + del_data.put("num_id", id) + DIST_COLL.remove(del_data) + } + static def saveDistrict(def m){ DBObject q = new BasicDBObject(); q.put("num_id",m.num_id) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index b5f86bf1b..1c287e355 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -92,8 +92,11 @@ class DataStandardBatch extends BaseBatch{ cityMap.each{city,alias-> println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) - districtMap.putAll(detailMap) +// if(city == "福州"){ + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) + districtMap.putAll(detailMap) +// } + } while(flag){ @@ -833,7 +836,7 @@ class DataStandardBatch extends BaseBatch{ } } } - if(!dbo){ + if(!dbo && isStandardRoad(copyObj.address)){ result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; @@ -920,7 +923,7 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) -// qdata.put("city","福州") + qdata.put("city","福州") // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 320e65b2f..4fe5dd4a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ } @Override void process(Object args) { - println SHORT_DATE + def citys = CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date()); From 3409e424811c656af1bba3b5d61569edfada5014 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 20 Jun 2023 13:36:22 +0800 Subject: [PATCH 03/22] update --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index f4e661538..3479e90ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 465eac28c..3f7cdff5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -428,7 +428,7 @@ class CcbCommonUtils { public static def findAllDistrictByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 From 9ed8f6dc21fc8bac36e4307352b024760526f2ae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 21 Jun 2023 15:10:18 +0800 Subject: [PATCH 04/22] update --- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 11 ++-- .../script/chongqing/HailanTest.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 27 +++++++++ .../script/district/MasterNameInAlias.groovy | 59 +++++++++++++++++++ .../fuzhou/DistrictInfoExportBatch.groovy | 2 + .../script/fuzhou/HouseExportBatch.groovy | 5 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 24 +++++--- 8 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index da8662b0e..492fb152d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); saveToPgSql(rows) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index e8bd49824..9156b9d6d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -45,6 +45,7 @@ class TradeStatLog { // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() + // task.deleteDistrict() // task.deleteDealData() // task.deleteDealData2() @@ -92,13 +93,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-02-01" - def date = "2023-03-01" - def endDate = "2023-04-01" - def newDate = "2023-05-01" + def startDate = "2023-03-01" + def date = "2023-04-01" + def endDate = "2023-05-01" + def newDate = "2023-06-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy index ff1b235f3..cdad0ddb6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -48,7 +48,7 @@ class HailanTest { } def run(){ - def inputfile = "D:\\数据\\微元素\\test.xlsx" + def inputfile = "D:\\数据\\微元素\\排查表.xlsx" // addThirdPrice(inputfile) // match(inputfile) // tradeAvgPrice(inputfile) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index bc5abebe7..e8cc8e9f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.commData +import com.fangjia.core.RegionData import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import groovy.json.JsonSlurper import net.sf.json.JSONArray @@ -3871,4 +3873,29 @@ class CommonDatautils { } + /** + * 根据经纬度获取区域 + * @param city + * @param location + * @return + */ + static def getRegionByLocation(city, def location){ + def json = GaoDeApiUtils.geocode(location); + def region = null; + if (json && json.info=="OK") { + def regeocode = json.regeocode + if(regeocode && regeocode.addressComponent){ + region = regeocode.addressComponent.district + if(!region){ + region = regeocode.addressComponent.township + } + } + } + if(region){ + region = RegionData.formatRegion(city, region) + } + return region; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy new file mode 100644 index 000000000..dd63a1c00 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.mongodb.BasicDBObject + +class MasterNameInAlias { + + static main(args) { + def task = new MasterNameInAlias() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def allList = [] + for(def city : citys){ + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"]) + + def cityList = [] + cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"] + for(def dist1 : disticts){ + def distList = [] + for(def dist2 : disticts){ + if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){ + if(dist2.alias && dist2.alias.contains(dist1.name)){ + def distance = '' + if(dist1.lng && dist2.lng){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d); + } + distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance] +// def newAlias = dist2.alias.remove(dist1.name) +// def d = new BasicDBObject() +// d.put("_id", dist2._id) +// +// def udata = new BasicDBObject() +// udata.put("alias", newAlias) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(d, new BasicDBObject('$set', udata)) + } + } + } + if(distList){ + cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', ''] + cityList.addAll(distList) +// cityList << [] + } + } + + if(cityList){ + allList.addAll(cityList) + println city +" "+cityList.size() + } + } + ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 464ac2301..eeabbfcae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.lang.StringUtils import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -321,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def s = str.toString(); index++; if(s && s!="null"){ + s = s.replace('"','\"') // if(s.toString().contains(",")){ // s = csvHandlerStr(s.toString()) // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 1dd114d8a..e4562341d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{ // sb.append(s+ "@|@") // } // } - sb.append(StringUtils.join(row.toArray(), "@|@")) + def str = StringUtils.join(row.toArray(), "@|@") + str = str.replace('"','\"') + sb.append(str) + sb.append(System.getProperty("line.separator")); println index diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 9ab7eacb3..07241353c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.paimai import com.fangjia.core.RegionData +import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.gj.ccb.FileUtils @@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -300,9 +301,7 @@ public class LianjiaCrawlByChrome { if(dealDate){ if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - println "1..."+new Date() def dbo = findTrade(durl , "链家在线") - println "2..."+new Date() if (!dbo) { parse(city, e, regionText, driver) } @@ -333,9 +332,7 @@ public class LianjiaCrawlByChrome { if(price){ def detailUrl = e.select(".title a").attr("href") driver.get(detailUrl) - println "3..."+new Date() def detailDoc = checkDocument(driver, detailUrl); - println "4..."+new Date() def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -353,6 +350,15 @@ public class LianjiaCrawlByChrome { if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } + if(!detail.region){ + def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1) + str = str.substring(0, str.indexOf("',")) + def locArrs = str.split(",") + def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble()) + if(gmap){ + detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat) + } + } def roomStr = arrs[1] if (roomStr) { @@ -427,9 +433,7 @@ public class LianjiaCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail - println "5..."+new Date() DEAL_PRICE_COLL.save(detail) - println "6..."+new Date() } } } @@ -581,9 +585,11 @@ public class LianjiaCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - elemUsername.sendKeys("15021384995"); +// elemUsername.sendKeys("15021384995"); +// elemPassword.sendKeys("12qwaszx"); - elemPassword.sendKeys("12qwaszx"); + elemUsername.sendKeys("16283166371"); + elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; Thread.sleep(15000) From 5a1e9a378a7e16a1cbb522b842f3e9238a00b809 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 21 Jun 2023 15:56:17 +0800 Subject: [PATCH 05/22] update --- .../com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy | 2 +- .../groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index eeabbfcae..75ff94eee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -322,7 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def s = str.toString(); index++; if(s && s!="null"){ - s = s.replace('"','\"') + s = s.replace('"','\\"') // if(s.toString().contains(",")){ // s = csvHandlerStr(s.toString()) // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index e4562341d..76925db82 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -115,7 +115,7 @@ class HouseExportBatch extends BaseBatch{ // } // } def str = StringUtils.join(row.toArray(), "@|@") - str = str.replace('"','\"') + str = str.replace('"','\\"') sb.append(str) sb.append(System.getProperty("line.separator")); From 27a18ce88d5a248cf3a4414844d558f780b75b21 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 21 Jun 2023 18:29:16 +0800 Subject: [PATCH 06/22] update --- .../ccbevaluation/BigdataDealBatch.groovy | 88 +++++++++---------- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +- .../fangjia/script/ccbevaluation/Test.groovy | 68 +++++++------- .../script/district/MasterNameInAlias.groovy | 18 ++-- 4 files changed, 89 insertions(+), 89 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 4d289d469..249590993 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -66,51 +66,51 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } println "curDate:"+curDate - def allCitys = CommonDatautils.findAllCity() - for(def city : allCitys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } +// } + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 -// for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// -// } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -// saveToPgSql(rows) + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt 生成 " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 91fb02cdc..591eb95bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-27") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) // // //更新静态数据表 -// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index db8fef0a1..2ce980c49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -33,8 +33,8 @@ import java.util.regex.*; def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -1287,18 +1287,18 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" -// -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['A']){ -//// def region = RegionData.formatRegion(d['A'], d['B']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") -//// if(dist){ -//// d['E'] = dist.region -//// d['F'] = dist._id.toString() -//// } -//// } +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx" + +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['A']){ + def region = RegionData.formatRegion(d['B'], d['C']) + if(region){ + def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") + if(dist){ + d['G'] = dist.region + d['H'] = dist._id.toString() + } + } // if(!d['F']){ // def dist = CommonDatautils.findDistrictById(d['A']) //// @@ -1308,30 +1308,30 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ // d['F'] = dist.status // } // } -// -// } -//},0) - - -def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx" -ExcelUtils.multiThreadingModify(fileName, { d -> - def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district") - if(count && count==1){ - def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district") - d['E'] = dist.city } - -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// d['B'] = dist.region -// d['C'] = dist.name -// d['J'] = dist.status -// } -// def newRegion = RegionData.formatRegion(d['A'], d['B']) -// d['F'] = newRegion },0) + + +//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx" +//ExcelUtils.multiThreadingModify(fileName, { d -> +// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district") +// if(count && count==1){ +// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district") +// d['E'] = dist.city +// } +// +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['J'] = dist.status +//// } +//// def newRegion = RegionData.formatRegion(d['A'], d['B']) +//// d['F'] = newRegion +//},0) + //ExcelUtils.modify(fileName, {d-> // if(d['A']){ // def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy index dd63a1c00..b604c38ff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy @@ -31,14 +31,14 @@ class MasterNameInAlias { distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d); } distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance] -// def newAlias = dist2.alias.remove(dist1.name) -// def d = new BasicDBObject() -// d.put("_id", dist2._id) -// -// def udata = new BasicDBObject() -// udata.put("alias", newAlias) -// udata.put("m_date", new Date()) -// DbUtils.seaweed.update(d, new BasicDBObject('$set', udata)) + dist2.alias.remove(dist1.name) + def d = new BasicDBObject() + d.put("_id", dist2._id) + + def udata = new BasicDBObject() + udata.put("alias", dist2.alias) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(d, new BasicDBObject('$set', udata)) } } } @@ -54,6 +54,6 @@ class MasterNameInAlias { println city +" "+cityList.size() } } - ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据") +// ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据") } } From 71ab29cee7925885b2013f2be3e7a93a3b41a73d Mon Sep 17 00:00:00 2001 From: "zhaojixiang@fangjia.com" Date: Thu, 6 Jul 2023 13:41:37 +0800 Subject: [PATCH 07/22] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=A5=BC=E6=A0=8B?= =?UTF-8?q?=E7=B3=BB=E6=95=B0=E7=AE=A1=E7=90=86=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/BuildingRatioController.java | 160 ++++++++++++++++++ .../bkoff/model/BuildingRatioModel.java | 125 ++++++++++++++ .../src/main/webapp/buildingRatio/add.ftl | 73 ++++++++ .../src/main/webapp/buildingRatio/edit.ftl | 74 ++++++++ .../src/main/webapp/buildingRatio/list.ftl | 108 ++++++++++++ .../src/main/webapp/layout/menu.ftl | 1 + .../dal/mongo/dao/bkoff/BuildingRatioDao.java | 12 ++ .../mongo/dao/bkoff/BuildingRatioDaoImpl.java | 12 ++ .../dal/mongo/model/bkoff/BuildingRatio.java | 95 +++++++++++ .../fangjia/service/bo/BuildingRatioBo.java | 85 ++++++++++ .../mongo/bkoff/BuildingRatioService.java | 18 ++ .../mongo/bkoff/BuildingRatioServiceImpl.java | 78 +++++++++ 12 files changed, 841 insertions(+) create mode 100644 projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java create mode 100644 projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java create mode 100644 projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl create mode 100644 projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl create mode 100644 projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDao.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDaoImpl.java create mode 100644 projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioService.java create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioServiceImpl.java diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java new file mode 100644 index 000000000..ad1eae98b --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java @@ -0,0 +1,160 @@ +package com.fangjia.bkoff.controllers; + +import com.fangjia.bkoff.model.BuildingRatioModel; +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; +import com.fangjia.service.bo.BuildingRatioBo; +import com.fangjia.service.mongo.bkoff.BuildingRatioService; +import org.apache.commons.lang.StringUtils; + +import java.net.URLEncoder; +import java.util.*; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioController { + + private BuildingRatioService buildingRatioService; + + /** + * 新建楼栋系数信息 + * + * @param model + * @return + */ + public Object doAddAction(BuildingRatioModel model) throws Exception { + if (StringUtils.isBlank(model.getCity_name()) || StringUtils.isBlank(model.getRegion()) + || StringUtils.isBlank(model.getDistrict()) || StringUtils.isBlank(model.getBuilding_name()) + || model.getRatio() == null) { + model.setMsg("城市,区域,小区,楼栋名称,楼栋系数不能为空"); + model.forward("/buildingRatio/add"); + return model; + } + try { + Date date = new Date(); + BuildingRatio buildingRatio = new BuildingRatio(); + buildingRatio.setCity(model.getCity_name()); + buildingRatio.setRegion(model.getRegion()); + buildingRatio.setDistrict(model.getDistrict()); + buildingRatio.setBuilding_name(model.getBuilding_name()); + buildingRatio.setRatio(model.getRatio()); + buildingRatio.setC_date(date); + buildingRatio.setM_date(date); + this.buildingRatioService.save(buildingRatio); + + model.setMsg("新建成功!"); + } catch (Exception e) { + e.printStackTrace(); + model.setMsg("新建失败!"); + } + model.redirect("/buildingRatio/add?msg=" + URLEncoder.encode(model.getMsg(), "UTF-8")); + return model; + } + + public Object doDeleteAction(BuildingRatioModel model){ + if(StringUtils.isNotBlank(model.getId())){ + this.buildingRatioService.deleteById(model.getId()); + } + model.setViewType("redirect"); + model.setViewParam("location","/buildingRatio/list"); + return model; + } + + /** + * 修改楼栋系数信息 + * + * @param model + * @return + */ + public Object updateAction(BuildingRatioModel model){ + String id = model.getId(); + if (StringUtils.isBlank(model.getCity_name()) || + StringUtils.isBlank(model.getRegion()) || + StringUtils.isBlank(model.getDistrict()) || + StringUtils.isBlank(model.getBuilding_name()) || + model.getRatio() == null) { + model.setMsg("城市,区域,小区,楼栋名称,楼栋系数不能为空"); + model.forward("/buildingRatio/edit"); + return model; + } + + Map data_buildingRatio = new HashMap(); + + data_buildingRatio.put("id", model.getId()); + data_buildingRatio.put("city", model.getCity_name()); + data_buildingRatio.put("region", model.getRegion()); + data_buildingRatio.put("district", model.getDistrict()); + data_buildingRatio.put("building_name", model.getBuilding_name()); + data_buildingRatio.put("ratio", model.getRatio()); + data_buildingRatio.put("m_date", new Date()); + + this.buildingRatioService.saveOrUpdateByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name(), data_buildingRatio); + + model.setViewType("redirect"); + model.setViewParam("location", "/buildingRatio/list"); + + return model; + } + + public Object listAction(BuildingRatioModel model) { + + List buildingRatioList = this.buildingRatioService.findAllByDistrict(model.getCity_name(),model.getRegion(),model.getDistrict()); + + List buildingRatioBoList = new ArrayList(); + + for (BuildingRatio buildingRatio : buildingRatioList) { + BuildingRatioBo bo = new BuildingRatioBo(); + + bo.setId(buildingRatio.getId()); + bo.setCity_name(buildingRatio.getCity()); + bo.setRegion(buildingRatio.getRegion()); + bo.setDistrict(buildingRatio.getDistrict()); + bo.setBuilding_name(buildingRatio.getBuilding_name()); + bo.setRatio(buildingRatio.getRatio()); + bo.setC_date(buildingRatio.getC_date()); + bo.setM_date(buildingRatio.getM_date()); + + buildingRatioBoList.add(bo); + } + + model.setBuildingRatioBoList(buildingRatioBoList); + return model; + } + + public Object addAction(BuildingRatioModel model) { + return model; + } + + /** + * 编辑api用户信息 + * + * @param model + * @return + */ + public Object editAction(BuildingRatioModel model) { + BuildingRatio buildingRatio = this.buildingRatioService.findBuildingRatioById(model.getId()); + + BuildingRatioBo bo = new BuildingRatioBo(); + bo.setId(buildingRatio.getId()); + bo.setCity_name(buildingRatio.getCity()); + bo.setRegion(buildingRatio.getRegion()); + bo.setDistrict(buildingRatio.getDistrict()); + bo.setBuilding_name(buildingRatio.getBuilding_name()); + bo.setRatio(buildingRatio.getRatio()); + + model.setBuildingRatioBo(bo); + if (StringUtils.isNotBlank(model.getMsg()) && model.getMsg().equals("success")) { + model.setMsg("更新成功!"); + } + return model; + } + + public BuildingRatioService getBuildingRatioService() { + return buildingRatioService; + } + + public void setBuildingRatioService(BuildingRatioService buildingRatioService) { + this.buildingRatioService = buildingRatioService; + } +} diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java new file mode 100644 index 000000000..ec8ed47b2 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java @@ -0,0 +1,125 @@ +package com.fangjia.bkoff.model; + +import com.fangjia.service.bo.BuildingRatioBo; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; +import java.util.List; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioModel extends BaseModel{ + + private String id; + private String city_name; + private String region; + private String district; + private String building_name; + private String building_id; + private Double ratio; + private String msg; + private List buildingRatioBoList; + private BuildingRatioBo buildingRatioBo; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date c_date; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date m_date; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCity_name() { + return city_name; + } + + public void setCity_name(String city_name) { + this.city_name = city_name; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district_name) { + this.district = district_name; + } + + public String getBuilding_name() { + return building_name; + } + + public void setBuilding_name(String building_name) { + this.building_name = building_name; + } + + public String getBuilding_id() { + return building_id; + } + + public void setBuilding_id(String building_id) { + this.building_id = building_id; + } + + public Double getRatio() { + return ratio; + } + + public void setRatio(Double ratio) { + this.ratio = ratio; + } + + public Date getC_date() { + return c_date; + } + + public void setC_date(Date c_date) { + this.c_date = c_date; + } + + public Date getM_date() { + return m_date; + } + + public void setM_date(Date m_date) { + this.m_date = m_date; + } + + public List getBuildingRatioBoList() { + return buildingRatioBoList; + } + + public void setBuildingRatioBoList(List buildingRatioBoList) { + this.buildingRatioBoList = buildingRatioBoList; + } + + public BuildingRatioBo getBuildingRatioBo() { + return buildingRatioBo; + } + + public void setBuildingRatioBo(BuildingRatioBo buildingRatioBo) { + this.buildingRatioBo = buildingRatioBo; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } +} diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl new file mode 100644 index 000000000..8b74961e8 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl @@ -0,0 +1,73 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 妤兼爧绯绘暟鏂板 + + <#include "/common_js.ftl" /> + + + + <#include "/layout/header.ftl" /> +
+${msg!''} + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ 鏍忎綅 + + 鍊� +
鍩庡競
鍖哄煙
灏忓尯鍚嶇О
妤兼爧鍚嶇О
妤兼爧绯绘暟
+
+
+ + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl new file mode 100644 index 000000000..a6f8f264c --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl @@ -0,0 +1,74 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 妤兼爧绯绘暟淇℃伅淇敼 + + + <#include "/common_js.ftl" /> + + + + + + <#include "/layout/header.ftl" /> +
+ ${msg!''} + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ 鏍忎綅 + + 鍊� +
鍩庡競
鍖哄煙
灏忓尯鍚嶇О
妤兼爧鍚嶇О
妤兼爧绯绘暟
+
+
+ + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl new file mode 100644 index 000000000..d6560fbaf --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl @@ -0,0 +1,108 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 鍧囦环绠$悊-妤兼爧绯绘暟绠$悊 + + <#include "/common_js.ftl" /> + + + <#include "/layout/header.ftl" /> + +
+
+ 鍩庡競锛� + 鍖哄煙锛� + 灏忓尯锛� + +
+
+ +
+ ${msg!''} + +
+
+
+ 鍏辨湁妤兼爧绯绘暟${buildingRatioBoList?size}涓� +
+ + + + + + + + + + + + + + + <#if (buildingRatioBoList??&&buildingRatioBoList?size>0)> + <#list buildingRatioBoList as bo> + + + + + + + + + + + + + +
+ ID + + 鍩庡競 + + 鍖哄煙 + + 灏忓尯鍚嶇О + + 妤兼爧鍚嶇О + + 妤兼爧绯绘暟 + + 鍒涘缓鏃ユ湡 + + 淇敼鏃ユ湡 + + 鎿嶄綔 +
+ ${bo.id!} + + ${bo.city_name!} + + ${bo.region!} + + ${bo.district!} + + ${bo.building_name!} + + ${bo.ratio!} + + <#if bo.c_date??> + ${bo.c_date?string("yyyy骞碝M鏈坉d鏃� HH:mm:ss")} + + + <#if bo.m_date??> + ${bo.m_date?string("yyyy骞碝M鏈坉d鏃� HH:mm:ss")} + + + 缂栬緫 + 鍒犻櫎 +
+
+
+ + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/layout/menu.ftl b/projects/fangjia-bkoff/src/main/webapp/layout/menu.ftl index 719e92e83..d44922973 100644 --- a/projects/fangjia-bkoff/src/main/webapp/layout/menu.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/layout/menu.ftl @@ -90,6 +90,7 @@
    <@menuitem uri='/avgPriceUpdate/doCreatTask' text="浣忓畢鍧囦环绠$悊" /> <@menuitem uri='/averagerConfig/list' text="浣忓畢绯绘暟绠$悊" /> + <@menuitem uri='/buildingRatio/list' text="妤兼爧绯绘暟绠$悊" /> <@menuitem uri='/avgPriceUpdate/listQR' text="浠锋牸璐ㄩ噺璺熻釜" /> <@menuitem uri='/avgPriceUpdate/listSQR' text="鏍锋湰璐ㄩ噺璺熻釜" /> <@menuitem uri='/comparePrice/listQR' text="璋冩煡鏁版嵁璺熻釜" /> diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDao.java new file mode 100644 index 000000000..f530cfff3 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDao.java @@ -0,0 +1,12 @@ +package com.fangjia.dal.mongo.dao.bkoff; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.bkoff.AveragerConfig; +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public interface BuildingRatioDao extends MongoDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDaoImpl.java new file mode 100644 index 000000000..192ad0456 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDaoImpl.java @@ -0,0 +1,12 @@ +package com.fangjia.dal.mongo.dao.bkoff; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.model.bkoff.AveragerConfig; +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioDaoImpl extends MongoDaoImpl implements BuildingRatioDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java new file mode 100644 index 000000000..70d186d25 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java @@ -0,0 +1,95 @@ +package com.fangjia.dal.mongo.model.bkoff; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; + + + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +@Entity +@Table(name = "business.building_ratio") +public class BuildingRatio implements MongoEntity { + + private String id; + private String city; + private String region; + private String district; + private String building_name; + + private Double ratio; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date c_date; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date m_date; + + @Id + @GeneratedValue + @Column(name = "_id") + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + @Column(name = "city") + public String getCity() { + return city; + } + public void setCity(String city) { + this.city = city; + } + @Column(name = "region") + public String getRegion() { + return region; + } + public void setRegion(String region) { + this.region = region; + } + @Column(name = "district") + public String getDistrict() { + return district; + } + public void setDistrict(String district) { + this.district = district; + } + @Column(name = "building_name") + public String getBuilding_name() { + return building_name; + } + public void setBuilding_name(String building_name) { + this.building_name = building_name; + } + @Column(name = "ratio") + public Double getRatio() { + return ratio; + } + public void setRatio(Double ratio) { + this.ratio = ratio; + } + @Column(name = "c_date") + public Date getC_date() { + return c_date; + } + public void setC_date(Date c_date) { + this.c_date = c_date; + } + @Column(name = "m_date") + public Date getM_date() { + return m_date; + } + public void setM_date(Date m_date) { + this.m_date = m_date; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java b/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java new file mode 100644 index 000000000..2dbb9d257 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java @@ -0,0 +1,85 @@ +package com.fangjia.service.bo; + +import java.io.Serializable; +import java.util.Date; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioBo implements Serializable { + + private static final long serialVersionUID = 1L; + private String id; + private String city_name; + private String region; + private String district; + private String building_name; + private Double ratio; + private Date c_date; + private Date m_date; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCity_name() { + return city_name; + } + + public void setCity_name(String city_name) { + this.city_name = city_name; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getBuilding_name() { + return building_name; + } + + public void setBuilding_name(String building_name) { + this.building_name = building_name; + } + + public Double getRatio() { + return ratio; + } + + public void setRatio(Double ratio) { + this.ratio = ratio; + } + + public Date getC_date() { + return c_date; + } + + public void setC_date(Date c_date) { + this.c_date = c_date; + } + + public Date getM_date() { + return m_date; + } + + public void setM_date(Date m_date) { + this.m_date = m_date; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioService.java new file mode 100644 index 000000000..422a90a41 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioService.java @@ -0,0 +1,18 @@ +package com.fangjia.service.mongo.bkoff; + +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; + +import java.util.List; +import java.util.Map; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public interface BuildingRatioService { + public void save(BuildingRatio buildingRatio); + public void deleteById(String id); + public void saveOrUpdateByBuildingName(String city, String region, String district, String building_name, Map data); + public BuildingRatio findBuildingRatioById(String id); + public List findAllByDistrict(String city, String region, String district); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioServiceImpl.java new file mode 100644 index 000000000..6ab6e1fd3 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioServiceImpl.java @@ -0,0 +1,78 @@ +package com.fangjia.service.mongo.bkoff; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.SortParam; +import com.fangjia.dal.mongo.dao.bkoff.BuildingRatioDao; +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; +import com.fangjia.dal.sql.model.FjtUsers; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioServiceImpl implements BuildingRatioService{ + + private BuildingRatioDao buildingRatioDao; + + @Override + public void save(BuildingRatio buildingRatio) { + + this.buildingRatioDao.save(buildingRatio); + } + + @Override + public void deleteById(String id) { + this.buildingRatioDao.deleteById(id); + } + @Override + public void saveOrUpdateByBuildingName(String city, String region, String district, String building_name, Map data) { + List params = new ArrayList(); + params.add(new QueryParam("city", city)); + params.add(new QueryParam("region", region)); + params.add(new QueryParam("district", district)); + params.add(new QueryParam("building_name", building_name)); + data.put("city", city); + data.put("region", region); + data.put("district", district); + data.put("building_name", building_name); + this.buildingRatioDao.updateByParams(data, params, true); + } + + @Override + public BuildingRatio findBuildingRatioById(String id) { + + return this.buildingRatioDao.findById(id); + } + + @Override + public List findAllByDistrict(String city, String region, String district) { + List params = new ArrayList(); + if(StringUtils.isNotBlank(city)){ + params.add(new QueryParam("city", city)); + } + if(StringUtils.isNotBlank(region)){ + params.add(new QueryParam("region", region)); + } + if(StringUtils.isNotBlank(district)){ + params.add(new QueryParam("district", district)); + } + + List sortParams = new ArrayList(); + sortParams.add(new SortParam("m_date", false)); + + return this.buildingRatioDao.findAllByParams(params, sortParams); + } + + public BuildingRatioDao getBuildingRatioDao() { + return buildingRatioDao; + } + + public void setBuildingRatioDao(BuildingRatioDao buildingRatioDao) { + this.buildingRatioDao = buildingRatioDao; + } +} From 674dfa732bf1244ab22ec1b7a9c42017a3875cfe Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 6 Jul 2023 18:06:14 +0800 Subject: [PATCH 08/22] =?UTF-8?q?script=20=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 118 ++-- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- .../DealAndThirdPriceDataProcess.groovy | 49 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +- .../ccbevaluation/PgsqlCommUtils.groovy | 10 + .../fangjia/script/ccbevaluation/Test.groovy | 89 ++- .../script/chongqing/HailanTest.groovy | 17 +- .../script/commData/CommonDatautils.groovy | 2 +- .../script/common/DataHandleUtils.groovy | 21 +- .../script/district/MasterNameInAlias.groovy | 4 +- .../script/fuzhou/AmapAddressUpdate.groovy | 616 ++++++++++++++++++ .../fuzhou/DistrictSellStatBatch.groovy | 2 +- .../com/fangjia/script/fuzhou/Test.groovy | 72 +- .../gj/district/FangjiaClientUtils.groovy | 19 + .../script/gj/district/SameDistName.groovy | 7 +- .../gj/location/DistrictLocationBatch.groovy | 121 ++++ .../script/gj/location/GaodeCityRange.groovy | 84 +++ ...roovy => RegionLocationUpdateBatch.groovy} | 140 +++- .../gj/{ => location}/RegionPoint.groovy | 41 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 20 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 19 +- .../fangjia/script/gj/poc/AddPriceTest.groovy | 34 +- .../script/linxsh/util/AddressUtil.groovy | 4 + .../script/looplineinfo/InsidePolygon.groovy | 47 +- .../temp/shenjian/util/HttpUtils.groovy | 40 ++ .../script/trade/TradeDealPriceStat.groovy | 20 +- .../script/wenyan/temp/DongGuanBank.groovy | 6 +- .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- 28 files changed, 1450 insertions(+), 164 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/GaodeCityRange.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/{LocationTest.groovy => RegionLocationUpdateBatch.groovy} (80%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/{ => location}/RegionPoint.groovy (82%) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 249590993..13fcabfc3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -33,7 +33,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")); + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); @@ -61,56 +61,58 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = CcbCommonUtils.getRunningCitys(args) +// def citys = CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } println "curDate:"+curDate -// def allCitys = CommonDatautils.findAllCity() -// for(def city : allCitys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } + def allCitys = ["福州"] //CommonDatautils.findAllCity() + for(def city : allCitys){ +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } + } + + +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 +// for(def city : citys){ +// index ++ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// // } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def index = 0 - for(def city : citys){ - index ++ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } - - } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成 " + rows.size() - - saveToPgSql(rows) +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成 " + rows.size() +// +// saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -649,16 +651,21 @@ class BigdataDealBatch extends BaseBatch{ def final AtomicInteger count_ = new AtomicInteger(0) for(def i=0;i + def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306] + def deal_pes = [] + def third_pes = [] + for(def d : dates){ + def map = findPeGteCity(row['B'], d) + if(map.deal_pe){ + deal_pes << map.deal_pe + } + if(map.third_pe){ + third_pes << map.third_pe + } + } + if(deal_pes){ + row['F'] = deal_pes.sum()/deal_pes.size() + } + if(third_pes){ + row['G'] = third_pes.sum()/third_pes.size() + } + println row['B'] + " "+row['F']+" "+row['G'] + }) + + } + + + + def findPeGteCity(cityName, yearmonth){ + def sql = "SELECT gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}" + def map = [:] + pgsqlInstance.eachRow(sql,{r-> + map.deal_pe = r.gujiamdltst_ppe15_pct + map.third_pe = r.gujiatrdpttstppe15pct + }); + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 591eb95bd..80f739637 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) // // //更新静态数据表 - updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") +// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy index f2a564cb7..2fd7e0c2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -17,4 +17,14 @@ class PgsqlCommUtils { }); return set } + + + public static def findAllDistrictCount(def cityid){ + def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def count = 0 + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 2ce980c49..0fd8eaa49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,8 +45,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -//def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", -// "postgres", "postgres", "org.postgresql.Driver") +def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") //def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"] //def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -1289,28 +1289,28 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx" -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['A']){ - def region = RegionData.formatRegion(d['B'], d['C']) - if(region){ - def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") - if(dist){ - d['G'] = dist.region - d['H'] = dist._id.toString() - } - } -// if(!d['F']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -//// +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['A']){ +// def region = RegionData.formatRegion(d['B'], d['C']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") // if(dist){ -//// d['H'] = dist.region -//// d['G'] = dist.name -// d['F'] = dist.status +// d['G'] = dist.region +// d['H'] = dist._id.toString() // } // } - - } -},0) +//// if(!d['F']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +////// +//// if(dist){ +////// d['H'] = dist.region +////// d['G'] = dist.name +//// d['F'] = dist.status +//// } +//// } +// +// } +//},0) @@ -1364,6 +1364,53 @@ def findGuidedPrice(sqlInstance, id){ return count } +def findPgDealPrice(c, pgsqlInstance, startDate){ + def sql = "select count(a.gujia_comm_id) from (SELECT distinct gujia_comm_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${c}' and txn_dt>='${startDate}') a" + def count = 0 + pgsqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + +citys = CcbCommonUtils.citys_all +def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"] +def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306] +def rows = [] +def headers = [] +headers << "城市" +headers.addAll(dates) +rows << headers +for(def c : citys){ + def row = [] + row << c + def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c)) +// for(def d : dates){ + def startDate = "2022-07-01" + def count = findPgDealPrice(c, pgsqlInstance, startDate) + row << count*100/disCount + +// def map = findSampleRate(pgsqlInstance, c, d) +// row << map.third +// } + rows << row +} +ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方") + + + + + +def findSampleRate(pgsqlInstance, cityName, yearmonth){ + def sql = "SELECT urbn_nm,gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}" + def map = [:] + pgsqlInstance.eachRow(sql,{r-> + map.deal = r.gujiamdltst_ppe15_pct + map.third = r.gujiatrdpttstppe15pct + }); + return map +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy index cdad0ddb6..25560ff03 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -48,7 +48,7 @@ class HailanTest { } def run(){ - def inputfile = "D:\\数据\\微元素\\排查表.xlsx" + def inputfile = "D:\\数据\\微元素\\排查表-价格对比-新.xlsx" // addThirdPrice(inputfile) // match(inputfile) // tradeAvgPrice(inputfile) @@ -58,7 +58,8 @@ class HailanTest { // addHouse(inputfile) // stat(inputfile) // hailanStat(inputfile) - addFzFjwPrice(inputfile) +// addFzFjwPrice(inputfile) + addFjwTobPrice(inputfile) } def addHouse(inputfile){ @@ -748,6 +749,18 @@ class HailanTest { }) } + + def addFjwTobPrice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(d['F']){ + def dbo = CommonDatautils.queryDistrictPrice("福州", d['F'], d['G'], DateUtils.parseFromTightByPattern("2023-07-02","yyyy-MM-dd")) + if(dbo){ + d['Q'] = dbo.avg_price_2b + } + } + }) + } + def addFzFjwPrice(inputfile){ ExcelUtils.modify(inputfile, { row -> if(row['B']) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e8cc8e9f8..9bcbde0b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -252,7 +252,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "长滩美墅三期") +// q.put("name", "燃灯小区") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy index 7fe3204dd..5edea094e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy @@ -250,8 +250,23 @@ class DataHandleUtils { return sum; } - - - + + + public static isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街道|大道|巷|胡同|街|坊]\\d+(-\\d+)?(号|弄)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } + + + + + static main(args) { + def a = isStandardRoad("杨桥西路236号") + println a + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy index b604c38ff..530d8b36d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy @@ -5,7 +5,7 @@ import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoUtils import com.mongodb.BasicDBObject - +//删除别名是另一个小区的主名 class MasterNameInAlias { static main(args) { @@ -14,7 +14,7 @@ class MasterNameInAlias { } def run(){ - def citys = CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity().reverse() def allList = [] for(def city : citys){ def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy new file mode 100644 index 000000000..cecc0536d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy @@ -0,0 +1,616 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.sql.ResultSetMetaData +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AmapAddressUpdate { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static main(args) { + def task = new AmapAddressUpdate() + def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx" + def addr = task.formatAddress("福州", "北三环路201号") + println addr +// task.run(path) +// task.similarMatch(path) +// task.addressRun(path) +// task.addressSource(path) +// task.amapRun() + +// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较 +// task.updateData(path) +// task.updateLocation(path)//更新经纬度 + } + def run(path){ + def citys = ["福州"]//CommonDatautils.findAllCity() + def allrets = [] + allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"] + for (def city : citys) { + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"]) + def rets = findAmapDetail(disticts) + allrets.addAll(rets) + } + ExcelUtils.write(path, allrets, "数据") + } + + def amapRun(){ + def city = "福州市" + def datas = findAllAmapByCity(city) +// datas = datas[0..10] + println "Es总量:"+datas.size() + def rets = match(datas) + + ExcelUtils.write("d:\\数据\\小区\\福州高德数据匹配.xlsx", rets, "数据") + } + + def match(datas){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + rets << ["ID","高德城市","高德区域","高德名称","高德地址","匹配城市","匹配区域","匹配名称","匹配类型","匹配ID"] + for(def i=0;i +// println data.adname+" "+data.name +" "+data.address +" "+data.type +// if(name == data.name && data.adname.contains(region) && data.type.contains("type")){ +// data.formatAddress = AddressUtil.formatAddress(city, data.address) +// loc = data +// } +// }) + return map; + } + + + + + + def addressRun(inputfile){ + + ExcelUtils.multiThreadingModify(inputfile, { row -> +// if(row['H'] && row['H'].contains("口步行") && row['H'].contains("(")){ +// row['I'] = row['H'].substring(0, row['H'].indexOf("(")) +// } + if(row['J']){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def distance = null + if(dist1){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, row['J'].toString().toDouble(), row['K'].toString().toDouble()); + } + row['M'] = distance + } + + def gaodeRoadFlag = false + if(row['I']){ + gaodeRoadFlag = DataHandleUtils.isStandardRoad(row['I']) + } + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(!row['N'] && row['D'] && row['I']){ + if(row['D']==row['I']){ + row['N'] = 100 + }else if(row['E'] && gaodeRoadFlag && row['E'].contains(row['I'])){ + row['N'] = 1 + } + } + + if(!row['N'] && !dbRoadFlag && gaodeRoadFlag){ + row['N'] = 2 + } + + if(!row['N'] && dbRoadFlag && gaodeRoadFlag && row['I'].contains(row['D'])){ + row['N'] = 3 + } + + if(!row['N'] && dbRoadFlag && gaodeRoadFlag){ + row['N'] = 4 + } + + }) + } + + def addressSource(inputfile) { + + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(!row['N'] && row['D'] && !row['O']){ + def addresses = findAllEsData(row['A'], row['B'], row['C']) + if(addresses){ + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(dbRoadFlag){ + def list = [] + list << row['D'] + if(row['E']){ + def arr = row['E'].split(" ") + list.addAll(arr as List) + } + for(def addr : list){ + if(addresses.contains(addr)){ + row['O'] = "诸葛找房" + break + } + } + } + } + } + }) + } + + + def findAllEsData(city,region, name){ + def rets = [] + def sql = "select * from community where city='${city}' and name='${name}' and source='诸葛找房APP'" + rets = findDatas(region, sql) + if(!rets){ + rets = findAliasDatas(city,region, name) + } + return rets; + } + + def findDatas(region, sql){ + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + if(d.region && d.region.contains(region) && d.address){ + rets << d.address + } + }) + return rets; + } + + + def findAliasDatas(city, region, name){ + def must = [] + def term1 = ["match":["city":city]] + must << term1 + + def should = [] + should << ["match":["alias":name]] + def params = ["query":["bool":["must":must, "should":should]]] + def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/community/_doc/_search"); + def rets = []; + if(s) { + def doc = new groovy.json.JsonSlurper().parseText(s) + def hits = doc.hits + if(hits){ + hits = hits.hits + for(def h : hits){ + def d = h._source + if(d.region && d.region.contains(region) && d.address && d.source=="诸葛找房APP"){ + rets << d.address + } + } + } + } + return rets; + } + + def addrAndLocationDistance(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(!row['N'] && row['D']){ + println row['D'] + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(dbRoadFlag){ + def gloJson = GaoDeApiUtils.geo(row['A'], row['D']) + if(gloJson) { + def glocation = null + if (gloJson && gloJson.get("geocodes")) { + def geocodes = gloJson.get("geocodes") + def map = geocodes.get(0) + if(map.formatted_address.contains(row['D'])){ + def arrs = map.location.split(",") + + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def distance = null + if(dist1){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, arrs[0].toString().toDouble(), arrs[1].toString().toDouble()); + } + row['P'] = distance + } + } + } + } + } + }) + } + + def updateData(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['Q'] && row['Q'].contains("修改地址")){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def udata = new BasicDBObject() + if(row['Q'].contains("可替换原地址")){ + udata.put("address", row['I']) + }else if(row['Q'].contains("可新增地址别名")){ + udata.put("address", row['I']) + def addrAlias = dist1.addr_alias + if(!addrAlias){ + addrAlias = [] + } + addrAlias << dist1.address + udata.put("addr_alias", addrAlias) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata)) + } + + + + }) + } + + def updateLocation(path){ + ExcelUtils.multiThreadingModify(path, { row -> + + if(row['R'] && row['R']=="是"){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + if(dist1){ + def udata = new BasicDBObject() + def lat = null + def lng = null + if(row['J']){ + lng = row['J'].toString().toDouble() + lat = row['K'].toString().toDouble() + }else{ + def gloJson = GaoDeApiUtils.geo(row['A'], row['D']) + if(gloJson) { + if (gloJson && gloJson.get("geocodes")) { + def geocodes = gloJson.get("geocodes") + def map = geocodes.get(0) + if (map.formatted_address.contains(row['D'])) { + def arrs = map.location.split(",") + lng = arrs[0].toString().toDouble() + lat = arrs[1].toString().toDouble() + } + } + } + } + + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + def lng2 = baidu.x + def lat2 = baidu.y + Long lng_new = lng *10000000000L + Long lat_new = lat *10000000000L + udata.put("lng", lng_new) + udata.put("lat", lat_new) + if(lng2){ + Long lng2_new = lng2 *10000000000L + Long lat2_new = lat2 *10000000000L + udata.put("lng2", lng2_new) + udata.put("lat2", lat2_new) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata)) + } + + } + + }) + } + + def similarMatch(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def d = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + if(d && d.lat){ + def dbo = getSimilarAmapLocation(d.city+"市", d.region, d.name, d.address, d.lat/10000000000.0, d.lng/10000000000.0) + if(dbo){ + row['F'] = dbo.region + row['G'] = dbo.name + row['H'] = dbo.address + row['I'] = formatAddress(d.city, dbo.address) + row['J'] = dbo.lng + row['K'] = dbo.lat + row['L'] = dbo.score + row['M'] = dbo.distance + row['S'] = dbo.type + } + } + } + + }) + } + + + + def getSimilarAmapLocation(city, region, name, address, lat, lng){ + + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "'") + sqlBuffer.append(" and MATCH(name,'${name}')") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + def maxSocreObj = null + while (rs.next()) { + def map = [:] + def mapName = rs.getString("name").replace("·","") + + println name +" "+ mapName + if(rs.getString("adname").contains(region)){ + map.name = rs.getString("name") + map.address = rs.getString("address") + + map.region = rs.getString("adname") + map.type = rs.getString("type") + map.match_type = "名称相似" + def location = rs.getObject("location") + if(location){ + map.lng = location.x + map.lat = location.y + } + + double nameScore = 0 + double addScore = 0 + double loctionScore = 0 + def distance = com.fangjia.core.util.GeoUtils.calculateDistance(lng, lat, map.lng.toString().toDouble(), map.lat.toString().toDouble()); + map.distance = distance + if(map.name && name){ + nameScore = CosineSimilarAlgorithm.getSimilarity(map.name, name); + } + if(map.address && address){ + addScore = CosineSimilarAlgorithm.getSimilarity(map.address, address); + } + + if(hasColumn(rs, "shape")){ + def shape = rs.getObject("shape") + if(shape){ + def coordinates = shape.getObject("coordinates") +// def pointList = InsidePolygon.createPoint2(points); + def flag = InsidePolygon.isInArea(lng, lat, coordinates) + if(flag){ + loctionScore = 0.5 + } + } + }else{ + + if(distance < 500 && (nameScore>0.5 || addScore>0.5)){ + loctionScore = 0.3 + } + } + + def score = nameScore*0.3 + addScore*0.2 + loctionScore + map.score = score + if(!maxSocreObj){ + if(score>0){ + map.formatAddress = AddressUtil.formatAddress(city, map.address) + maxSocreObj = map + } + }else{ + println score +" "+maxSocreObj.score + if(maxSocreObj && score>maxSocreObj.score){ + map.formatAddress = AddressUtil.formatAddress(city, map.address) + maxSocreObj = map + } + } + } + } + return maxSocreObj; + } + + + def hasColumn(ResultSet rs, String columnName) { + ResultSetMetaData rsmd = rs.getMetaData(); + int columns = rsmd.getColumnCount(); + for (int x = 1; x <= columns; x++) { + if (columnName.equals(rsmd.getColumnName(x))) { + return true; + } + } + return false; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index dc389ffe5..585d07a4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch { } def calcMonthlyPrice(){ - def citys = CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 9cea6cf74..b6eab8c46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -2,6 +2,9 @@ package com.fangjia.script.fuzhou import com.fangjia.core.RegionData import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.JsonUtils +import com.fangjia.core.util.Message +import com.fangjia.core.util.MessageInfo import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils import com.fangjia.script.util.DateUtils @@ -9,26 +12,71 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONObject import org.apache.commons.io.FileUtils import org.bson.types.ObjectId DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") +import com.fangjia.core.util.HttpRequest -ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> - def city = RegionData.formatCity("余姚") - d['D'] = city - if(city){ - d['E'] = RegionData.formatRegion(city, d['C']) - } -// if(!d['D']){ -// def citys = CommonDatautils.getCityByRegion(d['A']) -// if(citys && citys.size()==1){ -// d['F'] = citys.get(0) -// } +//Message message = new Message(); +//message.setMsgtype("text"); +//message.setText(new MessageInfo("测试")); +//URL url = new URL("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"); +//// 建立http连接 +//HttpURLConnection conn = (HttpURLConnection) url.openConnection(); +//conn.setDoOutput(true); +//conn.setDoInput(true); +//conn.setUseCaches(false); +//conn.setRequestMethod("POST"); +//conn.setRequestProperty("Charset", "UTF-8"); +//conn.setRequestProperty("Content-Type", "application/Json; charset=UTF-8"); +//conn.connect(); +//OutputStream out = conn.getOutputStream(); +//String textMessage = JsonUtils.toJson(message); +//byte[] data = textMessage.getBytes("UTF-8"); +//out.write(data); +//out.flush(); +//out.close(); +//System.out.println(conn.getResponseCode()); +//InputStream ins = conn.getInputStream(); +//byte[] data1 = new byte[ins.available()]; +//ins.read(data1); +//System.out.println(new String(data1)); + +//HttpRequest request = new HttpRequest() +//def params = [:] +//params.put("","") +//request.post("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430","", "utf-8") + + +//sendMarkdownMessage("租金", "测试") +//static void sendMarkdownMessage(String title, String msg){ +// HttpRequest request = new HttpRequest() +// JSONObject requestBody = new JSONObject() +// JSONObject markdown = new JSONObject() +// markdown.put("title", title) +// markdown.put("text", msg) +// requestBody.put("markdown", markdown) +// requestBody.put("msgtype", "markdown") +// request.postWithBody("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430", requestBody, null, "UTF-8") +//} +//ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> +// def city = RegionData.formatCity("余姚") +// d['D'] = city +// if(city){ +// d['E'] = RegionData.formatRegion(city, d['C']) // } -}) +// +//// if(!d['D']){ +//// def citys = CommonDatautils.getCityByRegion(d['A']) +//// if(citys && citys.size()==1){ +//// d['F'] = citys.get(0) +//// } +//// } +//}) //def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1) //def m = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index db058c5a7..a3236b53a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -1,9 +1,14 @@ package com.fangjia.script.gj.district import com.fangjia.core.util.HttpRequest +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.HttpRequestUtil +import com.mongodb.DBObject import net.sf.json.JSONArray import net.sf.json.JSONObject +import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang.StringUtils class FangjiaClientUtils { @@ -153,4 +158,18 @@ class FangjiaClientUtils { return columnValMap; } + + public static def saveOrUpdateDistrict(data) { + String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data) +// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr) +// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr) + + com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr) + if (json.status == 0) { + return "成功" + }else if(json.status == 1){ + return json.msg + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy index 7107037b9..597926900 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy @@ -10,6 +10,9 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +/** + *处理地址在地址别名中有重复 + */ class SameDistName { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") @@ -18,13 +21,13 @@ class SameDistName { def task = new SameDistName() // task.run() //地址和地址别名完全一致 -// task.addressPocess() + task.addressPocess() } def addressPocess(){ - def citys = CommonDatautils.findAllCity(); + def citys = ["福州"]//CommonDatautils.findAllCity(); for(def c : citys){ def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy new file mode 100644 index 000000000..c92e387fe --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy @@ -0,0 +1,121 @@ +package com.fangjia.script.gj.location + + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 判断小区是否在城市范围内 + */ +class DistrictLocationBatch { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static DBCollection RANGECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline") + + static main(args) { + def task = new DistrictLocationBatch() +// task.run() + task.deleteDist() + } + + def deleteDist(){ + ExcelUtils.multiThreadingModify("d:\\数据\\小区\\非本城市小区.xlsx",{d-> + DBObject q = new BasicDBObject() + q.put("city", d['A']) + q.put("region", d['B']) + q.put("name", d['C']) + q.put("cat", 'district') + q.put("status", 0) + + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }) + } + + def run() { + def citys = CommonDatautils.findAllCity() + def cityMap = findCityRange() + def allrets = [] + for (def city : citys) { + def points = cityMap.get(city) + def pointList = InsidePolygon.createPoint3(points); + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "lat", "lng"]) + def rets = findNotInCityDistricts(city, disticts, pointList) + allrets.addAll(rets) + } + ExcelUtils.write("d:\\数据\\小区\\非本城市小区.xlsx", allrets, "经纬度不在此城市中") + } + + def findNotInCityDistricts(city, districts, pointList){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + for(def i=0;i + if(!citys1.contains(c)){ + c = c.replace("州","") + JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") + if(resp.containsKey("districts")){ + def datas = resp.get("districts") + for(def d : datas){ + println c +" "+ d.name+" "+d.level + if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){ + rows << d + }else{ + if(d.level=="city"){ + rows << d + } + } + } + } + } + } + + + +// citys.each{c-> +// println c +// JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") +// if(resp.containsKey("districts")){ +// def datas = resp.get("districts") +// for(def d : datas){ +// println c +" "+ d.name+" "+d.level +// if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){ +// rows << d +// }else{ +// if(d.level=="city"){ +// rows << d +// } +// } +// } +// } +// } + if(rows){ + rows.each {r-> + def obj = new BasicDBObject() + obj.put("level", r.level) + obj.put("center", r.center) + obj.put("center", r.center) + obj.put("polyline", r.polyline) + obj.put("name", r.name) + obj.put("adcode", r.adcode) + obj.put("citycode", r.citycode) + + def city = RegionData.formatCity(r.name) + obj.put("fjw_city", city) + POLYLINECOLL.save(obj) + } + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy similarity index 80% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy index 3c2061a52..994019514 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy @@ -3,6 +3,9 @@ package com.fangjia.script.gj.location import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils @@ -25,28 +28,122 @@ import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern; - -class LocationTest { +/** + * 根据区域围栏修正区域 + */ +@BatchOwner(Watcher.GaoJuan) +class RegionLocationUpdateBatch extends BaseBatch{ static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { -// Date date=new Date(1622802609); -// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); -// String str = format.format(date); -// System.out.println(str); + runBatch(new RegionLocationUpdateBatch(), args); + } - def task = new LocationTest() - task.run() + @Override + public void process(Object args) { + regionUpdate() +// task.run() // task.run2() // task.run3() // task.run4() // task.run5() } + def regionUpdate(){ + def citys = CommonDatautils.findAllCity() + def rets = [] + def index = 0 + for(def city : citys){ + index ++ + def regionPointsMap = findRegionRange(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]); + def datas = findInRegionDistricts(city, districts, regionPointsMap) + println "【${index}】"+city +" 经纬度处理完成" + rets.addAll(datas) + } + ExcelUtils.write("D:\\数据\\小区\\区域纠正.xlsx", rets, "数据") + } + + + def findInRegionDistricts(city, districts, regionPointsMap){ + def index = 0 + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + def regionMap = [:] + regionPointsMap.each{k,v-> + def pointList = InsidePolygon.createPoint3(v); + regionMap.put(k, pointList) + } + def keys = regionPointsMap.keySet() + for(def i=0;i0){ def standardCity = RegionData.formatCity(cityMap.name) if(standardCity){ - saveData(standardCity, m.name, m.ad_code , ret); + def newRegion = RegionData.formatRegion(standardCity, m.name) + saveData(pro, standardCity, m.name, m.ad_code , ret, newRegion); }else{ println cityMap.name +" 没有标准化" } @@ -102,7 +103,7 @@ class RegionPointCrawl { def findCityCode(city){ Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE); def q = null; - if(city.contains("省") || city.endsWith("自治区")){ + if(city.contains("省") || city.endsWith("自治区") || ["上海市","天津市","北京市","重庆市"].contains(city)){ q = new BasicDBObject("province", pattern); }else{ q = new BasicDBObject("name", pattern); @@ -156,18 +157,22 @@ class RegionPointCrawl { } } - def saveData(city, region, code, ret){ + def saveData(pro, city, region, code, ret, formatRegion){ def q = new BasicDBObject() q.put("code", code) def dbo = POLYLINECOLL.findOne(q) if(!dbo){ def o = new BasicDBObject() + o.put("province", pro) o.put("city", city) o.put("region", region) o.put("code", code) o.put("center", ret.center.get(0)) o.put("level", ret.level.get(0)) o.put("polyline", ret.polyline.get(0)); + if(formatRegion){ + o.put("fjw_region", formatRegion) + } POLYLINECOLL.save(o); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index b190a324a..788d1703e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") //广州,佛山 重庆 @@ -204,8 +204,11 @@ public class BeiKeCrawlByChrome { if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl def dbo = findTrade(durl , "贝壳找房") + if (!dbo) { - parse(city, e, region, driver) + parse(city, e, region, driver, null) + }else if(dbo && !dbo.lift){ + parse(city, e, region, driver, dbo._id) } }else{ stop = true; @@ -224,7 +227,7 @@ public class BeiKeCrawlByChrome { return detailUrls } - def parse(city,e, region, driver){ + def parse(city,e, region, driver, id){ try { def detailUrl = e.select(".title a").attr("href") @@ -329,8 +332,11 @@ public class BeiKeCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail - DEAL_PRICE_COLL.save(detail) - + if(id){ + DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail)) + }else{ + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ @@ -459,9 +465,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") -// if (citys.contains(e.text())){ + if (citys.contains(e.text())){ list << [e.text(), href] -// } + } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 07241353c..6c0ce91f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -33,7 +33,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -303,7 +303,9 @@ public class LianjiaCrawlByChrome { detailUrls << durl def dbo = findTrade(durl , "链家在线") if (!dbo) { - parse(city, e, regionText, driver) + parse(city, e, regionText, driver, null) + }else if(dbo && !dbo.lift){ + parse(city, e, regionText, driver, dbo._id) } }else{ stop = true; @@ -326,7 +328,7 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e, regionText, driver){ + def parse(city,e, regionText, driver, id){ try { def price = e.select(".totalPrice .number").text() if(price){ @@ -433,7 +435,12 @@ public class LianjiaCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail - DEAL_PRICE_COLL.save(detail) + if(id){ + DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail)) + }else{ + DEAL_PRICE_COLL.save(detail) + } + } } } @@ -563,9 +570,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") -// if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityList << [e.text(), href] -// } + } } } return cityList diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index f419acf6d..67c803131 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -21,10 +22,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx" addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) +// addFjwPrice(filePath) } @@ -105,12 +107,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (d['B']) { -// def dist = CommonDatautils.findDistrictById(d['B']) -// if(dist){ -// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) - def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['G'] = gxdPrice -// } + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ + def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) +// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) + d['I'] = gxdPrice + } } }) } @@ -129,4 +131,22 @@ class AddPriceTest { } return price } + + + def addFjwPrice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(!d['K']){ + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ +// d['N'] = dist.region + def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd")) + if(dbo){ +// d['K'] = dbo.avg_price + d['K'] = dbo.avg_price_2b + } + } + } + }) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy index 517b6b634..61cc61f88 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy @@ -294,4 +294,8 @@ class AddressUtil { return result } + public static void main(String[] args){ + def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区") + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index fd646bf20..03f3e2b73 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -52,7 +52,40 @@ class InsidePolygon { return list; } - + + + + public static List createPoint3(String points){ + + List alllist = new ArrayList(); + if(points){ + List arrs = new ArrayList(); + if(points.contains("|")){ + String[] array = points.split("\\|"); + arrs.addAll(array) + }else{ + arrs.add(points) + } + + for(def arrStr : arrs){ + List list = new ArrayList(); + String[] array = arrStr.split(";"); + for(int i =0 ;i> getPoints(String points){ String[] array = points.split("-"); List> list = new ArrayList>(); @@ -227,4 +260,16 @@ class InsidePolygon { } return false; } + + + public static def isInArea(lng, lat, pointList){ + def flag = false; + for(def pointArr : pointList){ + flag = InsidePolygon.isPtInPoly(lng, lat, pointArr) + if(flag){ + break; + } + } + return flag; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy index ff11d6991..69c889afe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.temp.shenjian.util +import net.sf.json.JSONObject import org.apache.commons.httpclient.HttpClient import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.RequestEntity import org.apache.commons.httpclient.methods.StringRequestEntity /** @@ -26,4 +28,42 @@ class HttpUtils { } return false } + + + static def doPost2(String url, String jsonString){ + HttpClient httpClient = new HttpClient() + PostMethod postMethod = new PostMethod(url) + String responseMsg = null; + try{ + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + postMethod.setRequestEntity(requestEntity); + httpClient.executeMethod(postMethod); + InputStream is = postMethod.getResponseBodyAsStream(); + BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8")); + StringBuffer sb = new StringBuffer(""); + String str= ""; + while((str = br.readLine()) != null){ + sb.append(str); + } + responseMsg = sb.toString(); + } catch (Exception ex) { + ex.printStackTrace(); + } finally { + postMethod.releaseConnection(); + } + return JSONObject.fromObject(responseMsg); + } + + public String encodeHtml(String html) { + String mh=html; + try { + mh=new String(html.getBytes("iso-8859-1")); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + return mh; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy index c68ba4a85..67858c081 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy @@ -28,7 +28,25 @@ class TradeDealPriceStat { static main(args) { def task = new TradeDealPriceStat() - task.run() +// task.run() + task.deleteRepeat() + } + + def deleteRepeat(){ + def q = new BasicDBObject() + q.put("source", "我爱我家") + def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def set = new HashSet() + for(def c : cursor){ + def tradeDate = DateUtils.formatDateByPattern(c.trade_date, "yyyyMMdd") + def key = c.city +"#"+c.region+"#"+c.district_name+"#"+c.avg_price+"#"+tradeDate + if(set.contains(key)){ + DEAL_PRICE.remove(new BasicDBObject('_id', c._id)) + println c._id + }else{ + set << key + } + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy index b1a045e3f..520cf9f99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20230301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230401", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -38,6 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303") +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index bbc24f035..7b199ce7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220901", "yyyyMMdd") -def endDate = DateUtil.parseDate("20221001", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230201", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230301", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 From 02b9b2e156f10035bf9eb28d2160a65543b0b185 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 6 Jul 2023 18:15:04 +0800 Subject: [PATCH 09/22] =?UTF-8?q?=E5=9F=8E=E5=B8=82=20+=20=E5=8C=BA?= =?UTF-8?q?=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 9ed8300cc..5a38c237e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); -// cityRun(citys, startDate, endDate, cityCodeMap) -// regionRun(citys, startDate, endDate, cityCodeMap) + cityRun(citys, startDate, endDate, cityCodeMap) + regionRun(citys, startDate, endDate, cityCodeMap) districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } From 6b4b44319c295f5bbe16ff1e0e53e29b4352608b Mon Sep 17 00:00:00 2001 From: "zhaojixiang@fangjia.com" Date: Thu, 6 Jul 2023 18:59:18 +0800 Subject: [PATCH 10/22] =?UTF-8?q?=E6=A5=BC=E6=A0=8B=E7=B3=BB=E6=95=B0?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=A2=9E=E5=8A=A0=E5=88=9B=E5=BB=BA=E4=BA=BA?= =?UTF-8?q?=E5=92=8C=E5=A4=87=E6=B3=A8=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/BuildingRatioController.java | 16 ++++++++--- .../bkoff/model/BuildingRatioModel.java | 27 ++++++++++++------- .../src/main/webapp/buildingRatio/add.ftl | 8 ++++++ .../src/main/webapp/buildingRatio/edit.ftl | 8 ++++++ .../src/main/webapp/buildingRatio/list.ftl | 14 +++++++++- .../dal/mongo/model/bkoff/BuildingRatio.java | 18 +++++++++++++ .../fangjia/service/bo/BuildingRatioBo.java | 19 ++++++++++++- 7 files changed, 95 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java index ad1eae98b..76f7368fb 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java @@ -26,8 +26,8 @@ public class BuildingRatioController { public Object doAddAction(BuildingRatioModel model) throws Exception { if (StringUtils.isBlank(model.getCity_name()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getDistrict()) || StringUtils.isBlank(model.getBuilding_name()) - || model.getRatio() == null) { - model.setMsg("城市,区域,小区,楼栋名称,楼栋系数不能为空"); + || model.getRatio() == null || StringUtils.isBlank(model.getCreate_user())) { + model.setMsg("城市,区域,小区,楼栋名称,楼栋系数,创建人不能为空"); model.forward("/buildingRatio/add"); return model; } @@ -41,6 +41,8 @@ public class BuildingRatioController { buildingRatio.setRatio(model.getRatio()); buildingRatio.setC_date(date); buildingRatio.setM_date(date); + buildingRatio.setCreate_user(model.getCreate_user()); + buildingRatio.setNote(model.getNote()); this.buildingRatioService.save(buildingRatio); model.setMsg("新建成功!"); @@ -73,8 +75,8 @@ public class BuildingRatioController { StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getDistrict()) || StringUtils.isBlank(model.getBuilding_name()) || - model.getRatio() == null) { - model.setMsg("城市,区域,小区,楼栋名称,楼栋系数不能为空"); + model.getRatio() == null || StringUtils.isBlank(model.getCreate_user())) { + model.setMsg("城市,区域,小区,楼栋名称,楼栋系数,创建人不能为空"); model.forward("/buildingRatio/edit"); return model; } @@ -88,6 +90,8 @@ public class BuildingRatioController { data_buildingRatio.put("building_name", model.getBuilding_name()); data_buildingRatio.put("ratio", model.getRatio()); data_buildingRatio.put("m_date", new Date()); + data_buildingRatio.put("create_user", model.getCreate_user()); + data_buildingRatio.put("note", model.getNote()); this.buildingRatioService.saveOrUpdateByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name(), data_buildingRatio); @@ -114,6 +118,8 @@ public class BuildingRatioController { bo.setRatio(buildingRatio.getRatio()); bo.setC_date(buildingRatio.getC_date()); bo.setM_date(buildingRatio.getM_date()); + bo.setCreate_user(buildingRatio.getCreate_user()); + bo.setNote(buildingRatio.getNote()); buildingRatioBoList.add(bo); } @@ -142,6 +148,8 @@ public class BuildingRatioController { bo.setDistrict(buildingRatio.getDistrict()); bo.setBuilding_name(buildingRatio.getBuilding_name()); bo.setRatio(buildingRatio.getRatio()); + bo.setCreate_user(buildingRatio.getCreate_user()); + bo.setNote(buildingRatio.getNote()); model.setBuildingRatioBo(bo); if (StringUtils.isNotBlank(model.getMsg()) && model.getMsg().equals("success")) { diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java index ec8ed47b2..7afdeb9db 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java @@ -17,7 +17,8 @@ public class BuildingRatioModel extends BaseModel{ private String region; private String district; private String building_name; - private String building_id; + private String create_user; + private String note; private Double ratio; private String msg; private List buildingRatioBoList; @@ -67,14 +68,6 @@ public class BuildingRatioModel extends BaseModel{ this.building_name = building_name; } - public String getBuilding_id() { - return building_id; - } - - public void setBuilding_id(String building_id) { - this.building_id = building_id; - } - public Double getRatio() { return ratio; } @@ -122,4 +115,20 @@ public class BuildingRatioModel extends BaseModel{ public void setMsg(String msg) { this.msg = msg; } + + public String getCreate_user() { + return create_user; + } + + public void setCreate_user(String create_user) { + this.create_user = create_user; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } } diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl index 8b74961e8..dbfca71b2 100644 --- a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl @@ -50,6 +50,14 @@ 妤兼爧绯绘暟 + + 鍒涘缓浜� + + + + 澶囨敞 + + diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl index a6f8f264c..f11fc0866 100644 --- a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl @@ -55,6 +55,14 @@ 妤兼爧绯绘暟 + + 鍒涘缓浜� + + + + 澶囨敞 + + diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl index d6560fbaf..9945afa46 100644 --- a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl @@ -48,15 +48,21 @@ 妤兼爧鍚嶇О - + 妤兼爧绯绘暟 + + 鍒涘缓浜� + 鍒涘缓鏃ユ湡 淇敼鏃ユ湡 + + 澶囨敞 + 鎿嶄綔 @@ -83,6 +89,9 @@ ${bo.ratio!} + + ${bo.create_user!} + <#if bo.c_date??> ${bo.c_date?string("yyyy骞碝M鏈坉d鏃� HH:mm:ss")} @@ -93,6 +102,9 @@ ${bo.m_date?string("yyyy骞碝M鏈坉d鏃� HH:mm:ss")} + + ${bo.note!} + 缂栬緫 鍒犻櫎 diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java index 70d186d25..8868f4c4f 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java @@ -33,6 +33,8 @@ public class BuildingRatio implements MongoEntity { private Date c_date; @DateTimeFormat(pattern = "yyyy-MM-dd") private Date m_date; + private String create_user; + private String note; @Id @GeneratedValue @@ -92,4 +94,20 @@ public class BuildingRatio implements MongoEntity { public void setM_date(Date m_date) { this.m_date = m_date; } + @Column(name = "create_user") + public String getCreate_user() { + return create_user; + } + + public void setCreate_user(String create_user) { + this.create_user = create_user; + } + @Column(name = "note") + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java b/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java index 2dbb9d257..58f49d8da 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java @@ -18,7 +18,8 @@ public class BuildingRatioBo implements Serializable { private Double ratio; private Date c_date; private Date m_date; - + private String create_user; + private String note; public String getId() { return id; } @@ -82,4 +83,20 @@ public class BuildingRatioBo implements Serializable { public void setM_date(Date m_date) { this.m_date = m_date; } + + public String getCreate_user() { + return create_user; + } + + public void setCreate_user(String create_user) { + this.create_user = create_user; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } } From 86fd54d3923a4a76f2285d42d91f7a8f78e1875f Mon Sep 17 00:00:00 2001 From: "zhaojixiang@fangjia.com" Date: Fri, 7 Jul 2023 09:13:37 +0800 Subject: [PATCH 11/22] =?UTF-8?q?=E6=A5=BC=E6=A0=8B=E7=B3=BB=E6=95=B0?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=A2=9E=E5=8A=A0=E5=88=9B=E5=BB=BA=E4=BA=BA?= =?UTF-8?q?=E5=92=8C=E5=A4=87=E6=B3=A8=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl index f11fc0866..7324e748f 100644 --- a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl @@ -57,7 +57,7 @@ 鍒涘缓浜� - + 澶囨敞 From 4d156c4926cac98c75c765184bcdae1ef8a293a5 Mon Sep 17 00:00:00 2001 From: "zhaojixiang@fangjia.com" Date: Fri, 7 Jul 2023 09:24:04 +0800 Subject: [PATCH 12/22] =?UTF-8?q?=E6=A5=BC=E6=A0=8B=E7=B3=BB=E6=95=B0?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=A2=9E=E5=8A=A0=E5=88=9B=E5=BB=BA=E4=BA=BA?= =?UTF-8?q?=E5=92=8C=E5=A4=87=E6=B3=A8=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl index 7324e748f..4e058abd4 100644 --- a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl @@ -57,7 +57,7 @@ 鍒涘缓浜� - + 澶囨敞 From 0ee129219238f9c1f3b40db18b421c2c69671bff Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 7 Jul 2023 09:33:11 +0800 Subject: [PATCH 13/22] update --- .../fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy | 4 ++-- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 42feb5869..61b833dab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) -// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 - def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 +// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 5a38c237e..ed89f59ac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -63,7 +63,7 @@ class BigdataCitypriceBatch extends BaseBatch{ def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); cityRun(citys, startDate, endDate, cityCodeMap) regionRun(citys, startDate, endDate, cityCodeMap) - districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) +// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } def cityRun(citys, startDate, endDate, cityCodeDataMap){ From 6f79a0f83c43b1846a07ebb089ccbf3958e1bb2b Mon Sep 17 00:00:00 2001 From: "zhaojixiang@fangjia.com" Date: Fri, 7 Jul 2023 10:28:50 +0800 Subject: [PATCH 14/22] =?UTF-8?q?=E6=A5=BC=E6=A0=8B=E7=B3=BB=E6=95=B0?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=A2=9E=E5=8A=A0=E5=88=9B=E5=BB=BA=E4=BA=BA?= =?UTF-8?q?=E5=92=8C=E5=A4=87=E6=B3=A8=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/BuildingRatioController.java | 39 +++++++++---------- .../src/main/webapp/buildingRatio/add.ftl | 31 +++++++++++++++ .../src/main/webapp/buildingRatio/edit.ftl | 31 +++++++++++++++ .../mongo/bkoff/BuildingRatioService.java | 2 +- .../mongo/bkoff/BuildingRatioServiceImpl.java | 5 ++- 5 files changed, 86 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java index 76f7368fb..afd4fb55c 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java @@ -1,7 +1,9 @@ package com.fangjia.bkoff.controllers; +import com.fangjia.bkoff.model.ApiUserModel; import com.fangjia.bkoff.model.BuildingRatioModel; import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; +import com.fangjia.dal.sql.model.User; import com.fangjia.service.bo.BuildingRatioBo; import com.fangjia.service.mongo.bkoff.BuildingRatioService; import org.apache.commons.lang.StringUtils; @@ -24,14 +26,11 @@ public class BuildingRatioController { * @return */ public Object doAddAction(BuildingRatioModel model) throws Exception { - if (StringUtils.isBlank(model.getCity_name()) || StringUtils.isBlank(model.getRegion()) - || StringUtils.isBlank(model.getDistrict()) || StringUtils.isBlank(model.getBuilding_name()) - || model.getRatio() == null || StringUtils.isBlank(model.getCreate_user())) { - model.setMsg("城市,区域,小区,楼栋名称,楼栋系数,创建人不能为空"); - model.forward("/buildingRatio/add"); - return model; - } try { + if(checkBuildingNameAction(model)){ + model.redirect("/buildingRatio/add?msg=" + URLEncoder.encode(model.getMsg(), "UTF-8")); + return model; + } Date date = new Date(); BuildingRatio buildingRatio = new BuildingRatio(); buildingRatio.setCity(model.getCity_name()); @@ -69,18 +68,7 @@ public class BuildingRatioController { * @param model * @return */ - public Object updateAction(BuildingRatioModel model){ - String id = model.getId(); - if (StringUtils.isBlank(model.getCity_name()) || - StringUtils.isBlank(model.getRegion()) || - StringUtils.isBlank(model.getDistrict()) || - StringUtils.isBlank(model.getBuilding_name()) || - model.getRatio() == null || StringUtils.isBlank(model.getCreate_user())) { - model.setMsg("城市,区域,小区,楼栋名称,楼栋系数,创建人不能为空"); - model.forward("/buildingRatio/edit"); - return model; - } - + public Object updateAction(BuildingRatioModel model) throws Exception { Map data_buildingRatio = new HashMap(); data_buildingRatio.put("id", model.getId()); @@ -103,7 +91,7 @@ public class BuildingRatioController { public Object listAction(BuildingRatioModel model) { - List buildingRatioList = this.buildingRatioService.findAllByDistrict(model.getCity_name(),model.getRegion(),model.getDistrict()); + List buildingRatioList = this.buildingRatioService.findAllByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name()); List buildingRatioBoList = new ArrayList(); @@ -158,6 +146,17 @@ public class BuildingRatioController { return model; } + public boolean checkBuildingNameAction(BuildingRatioModel model) { + model.setViewType("json"); + model.setViewParam("config", null); + List buildingRatioList = buildingRatioService.findAllByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name()); + if (buildingRatioList.size()>0) { + model.setMsg("此楼栋名称已经存在!"); + return true; + } + return false; + } + public BuildingRatioService getBuildingRatioService() { return buildingRatioService; } diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl index dbfca71b2..ca7565298 100644 --- a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl @@ -67,6 +67,37 @@