From 6ff9f665b0bb204d31843058b0441b2a1e3e471c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sat, 17 Jul 2021 21:42:28 +0800 Subject: [PATCH] geng xin --- .../fangjia/script/gj/GaoDeMapCrawl.groovy | 2 - .../script/wenyan/auto/ReCalcAvgPrice.groovy | 32 +++++ .../wenyan/auto/SaveMapPoiToSeaweed.groovy | 113 ++++++++++++++++++ .../fangjia/script/wenyan/temp/QinLin.groovy | 8 +- .../wenyan/temp/SiChuanJingCheng.groovy | 6 +- .../fangjia/script/wenyan/temp/Test7.groovy | 85 ++++++++++++- 6 files changed, 233 insertions(+), 13 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index e93f98720..41bb9db8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -38,9 +38,7 @@ class GaoDeMapCrawl extends BaseBatch{ DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018) def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" - final def dbUrl = "mongodb://192.168.0.235:27018"; - ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) final AtomicInteger count = new AtomicInteger() def updateLists = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy new file mode 100644 index 000000000..5fceee582 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/ReCalcAvgPrice.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.wenyan.auto + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +//def keys = new BasicDBObject() +//keys.city = 1 +//keys.region = 1 +//keys.name = 1 +//keys.estate_type2 = 1 +//["重庆"].each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed-> +// BasicDBObject stat = DbUtils.findDistrictStat(seaweed, DateUtil.getFirstDayOfWeek(new Date())) +// if(!stat){ +// CommonUtils.recalculationAverage(seaweed.city, seaweed.region, seaweed.name, seaweed.estate_type2) +// println(seaweed) +// } +// }, keys) +// +//} +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\成交价——预测家.xlsx",{List item-> + BasicDBObject query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]) + BasicDBObject seaweed = DbUtils.seaweed.findOne(query) + if(seaweed){ + BasicDBObject stat = DbUtils.findDistrictStat(seaweed, item["E"]) + item["H"] = stat?.credit + } + println(item) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy new file mode 100644 index 000000000..b603f367e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/SaveMapPoiToSeaweed.groovy @@ -0,0 +1,113 @@ +package com.fangjia.script.wenyan.auto + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.apache.lucene.index.Term +import org.apache.lucene.search.BooleanClause +import org.apache.lucene.search.BooleanQuery +import org.apache.lucene.search.TermQuery + +/** + * 移除小区名中的区域信息 + * @param name + * @param region + * @return + */ +String removeRegion(String name, String region){ + if(name.startsWith(region) && name.length() - region.length() > 2){ + return name.replaceFirst(region, "") + } + return name; +} + +def String getStandardAddress(def city, def addr) { + if (StringUtils.isBlank(addr)) { + return null + } + JSONObject result = OpenApiClient.addressStandardize(city, addr) + if (result.containsKey("standardAddress") && result.getBoolean("standardAddress")) { + def standardAddr = result.getString("street") + if (result.containsKey("streetNumber")) { + standardAddr += result.getString("streetNumber") + } + return standardAddr + } + return null +} +def NOW = new Date() +def getType(String type){ + if(type == null){ + return "普通住宅" + } + if(type.contains("商住两用")){ + return "商住" + }else if(type.contains("别墅")){ + return "别墅" + }else{ + return "普通住宅" + } +} + +ExcelUtils.modify("D:\\nextcloud\\共享文档\\项目文档\\重庆农商行\\新增小区4.xlsx",{List item-> + if(item["C"] != "明佳园"){ + return + } + BasicDBObject seaweed = new BasicDBObject() + seaweed.city = item["A"] + seaweed.region = item["B"] + seaweed.name = item["C"] + def query = new BasicDBObject("city", seaweed.city).append("region", seaweed.region) + .append('name', seaweed.name) + .append("visible", 0) + .append("cat", "district"); + BasicDBObject one = DbUtils.seaweed.findOne(query) + if(!one){ + query.removeField("name") + query.append("alias", seaweed.name) + one = DbUtils.seaweed.findOne(query) + } + if (one && one.status == 0) { + item["i"] = "已存在" + return; + } + seaweed.address = getStandardAddress( seaweed.city, item["D"]) + String source = item["H"] + Double lng = item["E"] + Double lat = item["F"] + if(lng){ + if(source.contains("高德")){ + seaweed.lng = (long)(lng * 10000000000l) + seaweed.lat = (long)(lat * 10000000000l) + JSONObject geo = BaiDuApiUtils.googleToBaidu(lat, lng) + seaweed.lng2 = (long)(geo.lng * 10000000000l) + seaweed.lat2 = (long)(geo.lat * 10000000000l) + }else if(source.contains("百度")){ + Map point = GeoUtils.geoconvBaiduToGoogle(lng, lat); + long google_lat = (long)(point.get("lat") * 10000000000l); + long google_lng = (long)(point.get("lng") * 10000000000l); + long baiDu_lat = (long)(lat * 10000000000l) + long baiDu_lng = (long)(lng * 10000000000l) + seaweed.lng = google_lng + seaweed.lat = google_lat + seaweed.lat2 = baiDu_lat + seaweed.lng2 = baiDu_lng + } + } + seaweed.cat = "district" + seaweed.status = 0 + seaweed.c_date = NOW + seaweed.ip = "MACHINE_SAVE" + seaweed.estate_type2 = getType(item["G"]) + Map markMap = new HashMap() + seaweed.mark_map = markMap + markMap.put("data_source", "机器入库") + DbUtils.seaweed.save(seaweed) + println(seaweed) +}) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index e2790a624..ca851bcb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210401", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210501", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210501", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210601", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -22,10 +22,10 @@ while (startDate.before(endDate)) { calendar.add(Calendar.DATE, 1) def nextDate = calendar.getTime() count++ - OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + OPEN_URI_LOG.find(and(eq("user_id", "1007757"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) if (!hashSet.contains(result)) { - def item = ["5c6535b99c213017e0768ee1", "qlkj", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] data << item hashSet << result } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy index 08bf4c5b7..82cf47113 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210401", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210501", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210601", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210701", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -42,7 +42,7 @@ while (startDate.before(endDate)) { keys.addAll(params.keys()) def key = "${params.address}_${params.size}".toString() if (!hashSet.contains(key)) { - def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg] + def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg] param_keys.each { item << params[it] } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy index 168eaaf37..91e428cd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test7.groovy @@ -1,6 +1,11 @@ +import com.fangjia.core.util.CNNMFilter +import com.fangjia.core.util.HttpRequest import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import net.sf.json.JSONObject import org.apache.commons.io.FileUtils // @@ -35,7 +40,79 @@ import org.apache.commons.io.FileUtils // println("copy:" + file.getAbsolutePath() + "\tto:" + ("E:\\成交数据\\DBF汇总\\" + file.getName())) // } //} -DSCollection seawater = new DataStore("192.168.9.118", 27777).getCollection("fangjia.seawater") -seawater.find(new BasicDBObject("cat", "sell").append("city", "上海").append("region", "长宁").append("district_name", "新泾三村")).each { - println(it) -} \ No newline at end of file +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\副本地址标准化验证结果分析.xlsx",{List item-> +// HttpRequest request = new HttpRequest() +// JSONObject json = request.getJsonContent2("http://localhost:8082/structure", ["address": item["A"]], "UTF-8") +// if(json){ +// item["D"] = json.result.province +// item["E"] = json.result.city +// item["F"] = json.result.district +// item["G"] = json.result.dev +// item["H"] = json.result.township +// item["I"] = json.result.village +// item["J"] = json.result.group +// item["K"] = json.result.street +// item["L"] = json.result.streetNumber +// item["M"] = json.result.name +// item["N"] = json.result.buildingNumber +// item["O"] = json.result.unitNumber +// item["P"] = json.result.roomNumber +// item["q"] = json.result.floor +// item["r"] = json.result.formatAddress +// } +// println(item) +//}) +//for(int i=1; i < 100; i++){ +// println("${i}号楼#@#building_number") +// println("${i}栋#@#building_number") +// println("${i}幢#@#building_number") +// println("${i}单元#@#unit_number") +// if(i< 20){ +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}号楼#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}栋#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}幢#@#building_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}单元#@#unit_number") +// } +//} +//nums= "abcdefghijklmn" +//for(def num : nums.toCharArray()){ +// println("${num}栋#@#building_number") +// println("${num}幢#@#building_number") +// println("${num}单元#@#unit_number") +//} +//nums= "abcdefghijklmn" +//for(def num : nums.toCharArray()){ +// println("${num}座#@#building_number") +// for(int i = 1; i <11; i++){ +// println("${num}${i}座#@#building_number") +//// println("${num}${i}幢#@#building_number") +//// println("${num}${i}单元#@#unit_number") +// } +//} +//for(int i = 1; i <20; i++){ +// println("${i}区#@#part_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}区#@#part_number") +// println("${i}期#@#stage_number") +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}期#@#stage_number") +//} +// +//for(def num : "东南北中西甲乙丙丁".toCharArray()){ +// println("${num}区#@#part_number") +//} +//for(int i = -5; i < 100; i++){ +// println("${i}楼#@#floor") +// if(i < 10 && i > 0){ +// println("${CNNMFilter.convertArabicToCnNumeric(String.valueOf(i))}楼#@#floor") +// } +//} +//for(def num : "abcdefg东南西北".toCharArray()){ +// println("${num}户#@#room") +//} +//for(i in 1.. 40){ +// for(j in 1..9){ +// println("${i}0${j}室#@#room") +// } +//} +ElasticUtils.executeSqlQuery("select * from seawater where cat='sell' and district_name='新泾三村' and city='上海' and p_date > 20210101",{Map doc-> + println(doc.avg_price + "\t" + doc.root + "\t" + doc.p_date) +}) \ No newline at end of file