diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy similarity index 95% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy index dd8719a3f..934c3411d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.temp.shenjian +package com.fangjia.script.temp.shenjian.business import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -6,7 +6,6 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject import net.sf.json.JSONObject -import org.bson.types.ObjectId /** * 获取商圈地标 @@ -37,7 +36,6 @@ class BusinessDianpingAddr { } } } - collection.find() } static def parserByGaode(def city, def address) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy new file mode 100644 index 000000000..b18a14517 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy @@ -0,0 +1,95 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * 商圈地标导出 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingExport { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static void main(String[] args) { + def cityCodes = ["320500","610100","340100"]; + def cities = ["苏州", "西安", "合肥"] + def headers = ["城市", "区域", "商圈名称", "地标名称(3-5个)", "地标地址(与地标名称对应)", "地标经度(与地标名称对应)", "地标纬度(与地标名称对应)", "备注", "创建时间", "更新时间", "来源公司"]; + for (int i = 0; i < cities.size(); i++) { + final def city = cities[i]; + def result = [] + // 获取所有商圈信息 + def businessNames = collection.distinct("name", new BasicDBObject("city", city)) + // 获取商圈下3个地标 + for (def name : businessNames) { + List landmarks = new ArrayList<>() + List addresses = new ArrayList<>() + List lngs = new ArrayList<>() + List lats = new ArrayList<>() + String region = "" + String c_date + String m_date + int landmarkLen = 0; + collection.find(new BasicDBObject("city", city).append("name", name)).sort(new BasicDBObject("order", 1)).each { obj -> + // 按人气取经纬度不为空的地标 + if (landmarks.size() < 5) { + // 特殊标题去重 + if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1 + || String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1 + || String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1 + || String.valueOf(obj.landmark).indexOf(",") != -1) { + + } else { + // 防止导入导致字段过长,一个字符2个字节 + if (landmarkLen + String.valueOf(obj.landmark).length() <= 100) { + landmarkLen += String.valueOf(obj.landmark).length() + landmarks.add(obj.landmark) + addresses.add(obj.address) + lngs.add(obj.lng) + lats.add(obj.lat) + + if (StringUtils.isBlank(region)) { + region = obj.region + c_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss"); + m_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.m_date, "yyyy-MM-dd HH:mm:ss"); + } + } + } + } + } + def ret = [] + def city_code + def district_code + if (StringUtils.isNotBlank(region)) { + cityCollection.find(new BasicDBObject("fangjia_city", city).append("district", new BasicDBObject('$regex', region + "+")), + new BasicDBObject("city_code", 1).append("district_code", 1)).each {obj -> + city_code = obj.city_code + district_code = obj.district_code + } + } + ret[0] = city_code + ret[1] = district_code + ret[2] = name + ret[3] = StringUtils.join(landmarks, ",") + ret[4] = StringUtils.join(addresses, ",") + ret[5] = StringUtils.join(lngs, ",") + ret[6] = StringUtils.join(lats, ",") + ret[7] = "" + ret[8] = c_date + ret[9] = m_date + ret[10] = "fjw" + if (district_code != null) { + result << ret + } + } + FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy new file mode 100644 index 000000000..dd0c4c2dc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * 商圈数据修正 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingModify { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + + static void main(String[] args) { + def cityCodes = ["320500","610100","340100"]; + def city = "合肥" + def origin = "政务区" + def now = "蜀山区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + } +}