商圈地址优化
This commit is contained in:
+1
-3
@@ -1,4 +1,4 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
package com.fangjia.script.temp.shenjian.business
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -6,7 +6,6 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 鳳龰妀�華梓
|
||||
@@ -37,7 +36,6 @@ class BusinessDianpingAddr {
|
||||
}
|
||||
}
|
||||
}
|
||||
collection.find()
|
||||
}
|
||||
|
||||
static def parserByGaode(def city, def address) {
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
package com.fangjia.script.temp.shenjian.business
|
||||
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
|
||||
/**
|
||||
* 商圈地标导出
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/4/13
|
||||
*/
|
||||
class BusinessDianpingExport {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
|
||||
static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
|
||||
|
||||
static void main(String[] args) {
|
||||
def cityCodes = ["320500","610100","340100"];
|
||||
def cities = ["苏州", "西安", "合肥"]
|
||||
def headers = ["城市", "区域", "商圈名称", "地标名称(3-5个)", "地标地址(与地标名称对应)", "地标经度(与地标名称对应)", "地标纬度(与地标名称对应)", "备注", "创建时间", "更新时间", "来源公司"];
|
||||
for (int i = 0; i < cities.size(); i++) {
|
||||
final def city = cities[i];
|
||||
def result = []
|
||||
// 获取所有商圈信息
|
||||
def businessNames = collection.distinct("name", new BasicDBObject("city", city))
|
||||
// 获取商圈下3个地标
|
||||
for (def name : businessNames) {
|
||||
List<String> landmarks = new ArrayList<>()
|
||||
List<String> addresses = new ArrayList<>()
|
||||
List<String> lngs = new ArrayList<>()
|
||||
List<String> lats = new ArrayList<>()
|
||||
String region = ""
|
||||
String c_date
|
||||
String m_date
|
||||
int landmarkLen = 0;
|
||||
collection.find(new BasicDBObject("city", city).append("name", name)).sort(new BasicDBObject("order", 1)).each { obj ->
|
||||
// 按人气取经纬度不为空的地标
|
||||
if (landmarks.size() < 5) {
|
||||
// 特殊标题去重
|
||||
if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1
|
||||
|| String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1
|
||||
|| String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1
|
||||
|| String.valueOf(obj.landmark).indexOf(",") != -1) {
|
||||
|
||||
} else {
|
||||
// 防止导入导致字段过长,一个字符2个字节
|
||||
if (landmarkLen + String.valueOf(obj.landmark).length() <= 100) {
|
||||
landmarkLen += String.valueOf(obj.landmark).length()
|
||||
landmarks.add(obj.landmark)
|
||||
addresses.add(obj.address)
|
||||
lngs.add(obj.lng)
|
||||
lats.add(obj.lat)
|
||||
|
||||
if (StringUtils.isBlank(region)) {
|
||||
region = obj.region
|
||||
c_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss");
|
||||
m_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.m_date, "yyyy-MM-dd HH:mm:ss");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def ret = []
|
||||
def city_code
|
||||
def district_code
|
||||
if (StringUtils.isNotBlank(region)) {
|
||||
cityCollection.find(new BasicDBObject("fangjia_city", city).append("district", new BasicDBObject('$regex', region + "+")),
|
||||
new BasicDBObject("city_code", 1).append("district_code", 1)).each {obj ->
|
||||
city_code = obj.city_code
|
||||
district_code = obj.district_code
|
||||
}
|
||||
}
|
||||
ret[0] = city_code
|
||||
ret[1] = district_code
|
||||
ret[2] = name
|
||||
ret[3] = StringUtils.join(landmarks, ",")
|
||||
ret[4] = StringUtils.join(addresses, ",")
|
||||
ret[5] = StringUtils.join(lngs, ",")
|
||||
ret[6] = StringUtils.join(lats, ",")
|
||||
ret[7] = ""
|
||||
ret[8] = c_date
|
||||
ret[9] = m_date
|
||||
ret[10] = "fjw"
|
||||
if (district_code != null) {
|
||||
result << ret
|
||||
}
|
||||
}
|
||||
FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8")
|
||||
}
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.fangjia.script.temp.shenjian.business
|
||||
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
|
||||
/**
|
||||
* 商圈数据修正
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/4/13
|
||||
*/
|
||||
class BusinessDianpingModify {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
|
||||
|
||||
static void main(String[] args) {
|
||||
def cityCodes = ["320500","610100","340100"];
|
||||
def city = "合肥"
|
||||
def origin = "政务区"
|
||||
def now = "蜀山区"
|
||||
collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj ->
|
||||
collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user