商圈地址优化

This commit is contained in:
沈健
2021-04-13 19:50:58 +08:00
parent d6f7803c6c
commit a4717f4e83
3 changed files with 125 additions and 3 deletions
@@ -1,4 +1,4 @@
package com.fangjia.script.temp.shenjian
package com.fangjia.script.temp.shenjian.business
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -6,7 +6,6 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.bson.types.ObjectId
/**
* 鳳龰妀�華梓
@@ -37,7 +36,6 @@ class BusinessDianpingAddr {
}
}
}
collection.find()
}
static def parserByGaode(def city, def address) {
@@ -0,0 +1,95 @@
package com.fangjia.script.temp.shenjian.business
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
/**
* 商圈地标导出
*
* @author shenjian
* @since 2021/4/13
*/
class BusinessDianpingExport {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static void main(String[] args) {
def cityCodes = ["320500","610100","340100"];
def cities = ["苏州", "西安", "合肥"]
def headers = ["城市", "区域", "商圈名称", "地标名称(3-5个)", "地标地址(与地标名称对应)", "地标经度(与地标名称对应)", "地标纬度(与地标名称对应)", "备注", "创建时间", "更新时间", "来源公司"];
for (int i = 0; i < cities.size(); i++) {
final def city = cities[i];
def result = []
// 获取所有商圈信息
def businessNames = collection.distinct("name", new BasicDBObject("city", city))
// 获取商圈下3个地标
for (def name : businessNames) {
List<String> landmarks = new ArrayList<>()
List<String> addresses = new ArrayList<>()
List<String> lngs = new ArrayList<>()
List<String> lats = new ArrayList<>()
String region = ""
String c_date
String m_date
int landmarkLen = 0;
collection.find(new BasicDBObject("city", city).append("name", name)).sort(new BasicDBObject("order", 1)).each { obj ->
// 按人气取经纬度不为空的地标
if (landmarks.size() < 5) {
// 特殊标题去重
if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1
|| String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1
|| String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1
|| String.valueOf(obj.landmark).indexOf(",") != -1) {
} else {
// 防止导入导致字段过长,一个字符2个字节
if (landmarkLen + String.valueOf(obj.landmark).length() <= 100) {
landmarkLen += String.valueOf(obj.landmark).length()
landmarks.add(obj.landmark)
addresses.add(obj.address)
lngs.add(obj.lng)
lats.add(obj.lat)
if (StringUtils.isBlank(region)) {
region = obj.region
c_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss");
m_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.m_date, "yyyy-MM-dd HH:mm:ss");
}
}
}
}
}
def ret = []
def city_code
def district_code
if (StringUtils.isNotBlank(region)) {
cityCollection.find(new BasicDBObject("fangjia_city", city).append("district", new BasicDBObject('$regex', region + "+")),
new BasicDBObject("city_code", 1).append("district_code", 1)).each {obj ->
city_code = obj.city_code
district_code = obj.district_code
}
}
ret[0] = city_code
ret[1] = district_code
ret[2] = name
ret[3] = StringUtils.join(landmarks, ",")
ret[4] = StringUtils.join(addresses, ",")
ret[5] = StringUtils.join(lngs, ",")
ret[6] = StringUtils.join(lats, ",")
ret[7] = ""
ret[8] = c_date
ret[9] = m_date
ret[10] = "fjw"
if (district_code != null) {
result << ret
}
}
FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8")
}
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.temp.shenjian.business
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
/**
* 商圈数据修正
*
* @author shenjian
* @since 2021/4/13
*/
class BusinessDianpingModify {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
static void main(String[] args) {
def cityCodes = ["320500","610100","340100"];
def city = "合肥"
def origin = "政务区"
def now = "蜀山区"
collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj ->
collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true)
}
}
}