Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2022-05-27 15:17:06 +08:00
7 changed files with 480 additions and 61 deletions
@@ -30,7 +30,7 @@ import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* oralce中小区统计数据同步至85中
* oracle中小区统计数据同步至85中
*/
class OracleDistrict2Mongo85 {
@@ -0,0 +1,52 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DistrictCodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
/**
* oracle中小区置信度数据同步至85中
*/
class OracleDistrictBasePrice2Mongo85 {
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017)
static main(args) {
def sql = "SELECT cityname, distname, commid, commname, confidence FROM community_base_price_gj WHERE YEAR_MONTH='202205'"
def count = 0
sqlInstance.eachRow(sql, { r ->
def city = r['cityname']
def region = RegionData.formatRegion(city, r['distname'])
DBObject dbObject = new BasicDBObject("_id", r['commid'])
if (r['confidence'] != null) {
def credit = new BigDecimal(r['confidence']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble()
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
.append("name", r['commname'])
.append("credit", credit)
.append("c_date", new Date());
communityBasePriceConn.update(dbObject, updateObject, true, false)
count++;
} else {
print "置信度为空:" + r.toString()
}
if (count % 500 == 0) {
println "已写入" + count + "条小区数据"
}
});
}
}
@@ -13,7 +13,13 @@ import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.model.Filters
import com.mongodb.client.model.UpdateOneModel
import com.mongodb.client.model.UpdateOptions
import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils
import org.bson.Document
import org.bson.conversions.Bson
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
@@ -27,74 +33,85 @@ import java.util.concurrent.Executors
*/
class SeaweedAroundPoiSingal {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
def types = listAllType()
//def types = "0601"
DBObject query = new BasicDBObject()
query.put("status", 1)
conn136.find(query).batchSize(30).each {objTemp ->
final def obj = objTemp
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
getNewSeaweed()
// def types = listAllType()
// //def types = "0601"
// DBObject query = new BasicDBObject()
// query.put("status", 1)
// conn136.find(query).batchSize(30).each {objTemp ->
// final def obj = objTemp
// def city = obj.city
// def lng = obj.lng
// def lat = obj.lat
// List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
// if (CollectionUtils.isNotEmpty(pois)) {
// String jsonStr = JSONObject.toJSONString(pois)
// HttpUtils.postForJson(URL, jsonStr)
// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
// } else {
// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
// }
// println("已处理小区:" + obj)
// }
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("alias", 1)
filter.put("address", 1)
filter.put("addr_alias", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List list = []
List<WriteModel<Document>> writeModelList = new ArrayList<>()
def count = 0
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
// 已存在小区,无需同步
} else {
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
Document updateBson = new Document()
Document update = new Document()
update.put("city", obj.city)
update.put("region", obj.region)
update.put("name", obj.name)
update.put("alias", obj.alias)
update.put("address", obj.address)
update.put("addr_alias", obj.addr_alias)
try {
update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
}
update.put("status", 1)
update.put("flag", "未精确获取高德信息")
updateBson.append('$set', update)
list << newObj
if (list.size() == 5000) {
conn136.insert(list)
list.clear()
}
Bson filterBson = Filters.eq("_id", obj._id)
UpdateOptions updateOptions = new UpdateOptions();
updateOptions.upsert(true)
UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
writeModelList.add(updateOneModel)
if (writeModelList.size() == 5000) {
conn136.bulkWrite(writeModelList)
count += 5000
println(count)
writeModelList.clear()
}
}
conn136.insert(list)
list.clear()
if (writeModelList.size() > 0) {
conn136.bulkWrite(writeModelList)
writeModelList.clear()
}
}
static def getByIdAndInsert(String id) {
@@ -0,0 +1,160 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.model.Filters
import com.mongodb.client.model.UpdateOneModel
import com.mongodb.client.model.UpdateOptions
import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.bson.conversions.Bson
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区POI信息精确查找
*
* @author shenjian
* @since 2022/5/25
*/
class SeaweedPoiDetail {
static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static final ExecutorService executorService = Executors.newFixedThreadPool(20)
static void main(String[] args) {
DBObject query = new BasicDBObject()
query.put("status", 1)
conn136.find(query).batchSize(1000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def city = obj.city
def region = obj.region
def name = obj.name
def alias = obj.alias
def address = obj.address
def addr_alias = obj.addr_alias
StringJoiner stringJoiner = new StringJoiner("|")
// 主小区名搜索
stringJoiner.add(name)
// 根据别名搜
if (alias != null && CollectionUtils.isNotEmpty(alias)) {
for (String str : alias) {
stringJoiner.add(str)
}
}
// 根据路名搜
if (StringUtils.isNotBlank(address)) {
stringJoiner.add(address)
}
// 根据路别名搜
if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) {
for (String str : addr_alias) {
stringJoiner.add(str)
}
}
String keywords = stringJoiner.toString()
List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes)
if (CollectionUtils.isEmpty(pois)) {
pois = GaoDeUtils.searchPoi(city, name, region, typecodes)
}
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
// for (AmapPoi amapPoi : pois) {
// List<AmapPoiDetail> amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId());
// if (CollectionUtils.isNotEmpty(amapPoiList)) {
// detailList.addAll(amapPoiList)
// }
// }
if (CollectionUtils.isNotEmpty(detailList)) {
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到")))
} else {
conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区")))
}
println("已处理小区:" + obj)
}
})
}
executorService.shutdown()
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("alias", 1)
filter.put("address", 1)
filter.put("addr_alias", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List<WriteModel<Document>> writeModelList = new ArrayList<>()
def count = 0
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
Document updateBson = new Document()
Document update = new Document()
update.put("city", obj.city)
update.put("region", obj.region)
update.put("name", obj.name)
update.put("alias", obj.alias)
update.put("address", obj.address)
update.put("addr_alias", obj.addr_alias)
try {
update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
}
update.put("status", 1)
update.put("flag", "未精确获取高德信息")
updateBson.append('$set', update)
Bson filterBson = Filters.eq("_id", obj._id)
UpdateOptions updateOptions = new UpdateOptions();
updateOptions.upsert(true)
UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
writeModelList.add(updateOneModel)
if (writeModelList.size() == 5000) {
conn136.bulkWrite(writeModelList)
count += 5000
println(count)
writeModelList.clear()
}
}
if (writeModelList.size() > 0) {
conn136.bulkWrite(writeModelList)
writeModelList.clear()
}
}
}
@@ -0,0 +1,40 @@
package com.fangjia.script.temp.shenjian.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
/**
* 通用实体与DTO互相转换工具类
*
* @author shenjian
* @date 2021/06/16
*/
public class CommonDtoUtils {
public static <T> T transformV(Class<T> targetClass, Object source) {
if (source == null) {
return null;
} else {
try {
String jsonSource = JSON.toJSONString(source);
return JSONObject.parseObject(jsonSource, targetClass);
} catch (Exception var3) {
throw var3;
}
}
}
public static <T> List<T> transformListV(Class<T> targetClass, List<?> listSource) {
if (listSource == null) {
return null;
} else {
try {
String jsonSource = JSON.toJSONString(listSource);
return JSONArray.parseArray(jsonSource, targetClass);
} catch (Exception var3) {
throw var3;
}
}
}
}
@@ -7,10 +7,12 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import org.apache.commons.lang.StringUtils
import org.openqa.selenium.json.Json
import java.util.concurrent.atomic.AtomicInteger
@@ -23,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger
class GaoDeUtils {
// 新获取key
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"]
def static GAO_DE_AK = ["4ccfe3c898a9489884bd2bcb1ef6dcde", "4e9592208336433ee1b7aa4d1d434418"]
def static AtomicInteger keyUsed = new AtomicInteger(0)
def static codeNameMap = new HashMap<String, String>(){{
@@ -88,12 +90,25 @@ class GaoDeUtils {
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
result.addAll(DataUtil.toMongoDbObj(json.pois))
} else if (json && json.infocode && json.infocode == 10003) {
throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
} else if (json && json.infocode && json.infocode == 10004) {
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
break;
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
} else {
break
}
@@ -146,7 +161,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -160,7 +175,9 @@ class GaoDeUtils {
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
//System.exit(1)
Thread.sleep(60000)
break;
} else {
break
}
@@ -222,12 +239,65 @@ class GaoDeUtils {
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
//System.exit(1)
Thread.sleep(60000)
}
return Collections.emptyList()
}
/**
* 获取详情
*/
static List<AmapPoiDetail> getAmapDetailById(id){
def params = [:]
params.key = GAO_DE_AK[keyUsed.intValue()]
params.id = id
params.output = 'json'
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
//System.exit(1)
Thread.sleep(60000)
}
return Collections.emptyList()
}
@@ -270,7 +340,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -278,13 +348,13 @@ class GaoDeUtils {
}
break;
}
} else if (json && json.infocode && json.infocode == "10004") {
} else if (json && json.infocode && (json.infocode == "10004" || json.infocode == "10021" || json.infocode == "10017")) {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
} else if (json && json.infocode && (json.infocode == "20000")) {
// 无效参数
break;
}
}
return result
@@ -326,7 +396,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -367,7 +437,16 @@ class GaoDeUtils {
return ["lng": arrs[0], "lat": arrs[1]]
}
} else if (json && json.infocode && json.infocode == "10044") {
System.exit(1)
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
@@ -0,0 +1,71 @@
package com.fangjia.script.statistics.poi;
import java.util.List;
/**
* @author shenjian
* @since 2022/5/25
*/
@lombok.NoArgsConstructor
@lombok.Data
public class AmapPoiDetail {
@com.alibaba.fastjson.annotation.JSONField(name = "id")
private String id;
@com.alibaba.fastjson.annotation.JSONField(name = "parent")
private List<?> parent;
@com.alibaba.fastjson.annotation.JSONField(name = "childtype")
private List<?> childtype;
@com.alibaba.fastjson.annotation.JSONField(name = "name")
private String name;
@com.alibaba.fastjson.annotation.JSONField(name = "type")
private String type;
@com.alibaba.fastjson.annotation.JSONField(name = "typecode")
private String typecode;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_type")
private List<?> bizType;
@com.alibaba.fastjson.annotation.JSONField(name = "address")
private String address;
@com.alibaba.fastjson.annotation.JSONField(name = "location")
private String location;
@com.alibaba.fastjson.annotation.JSONField(name = "tel")
private String tel;
@com.alibaba.fastjson.annotation.JSONField(name = "pname")
private String pname;
@com.alibaba.fastjson.annotation.JSONField(name = "cityname")
private String cityname;
@com.alibaba.fastjson.annotation.JSONField(name = "adname")
private String adname;
@com.alibaba.fastjson.annotation.JSONField(name = "importance")
private List<?> importance;
@com.alibaba.fastjson.annotation.JSONField(name = "shopid")
private List<?> shopid;
@com.alibaba.fastjson.annotation.JSONField(name = "shopinfo")
private String shopinfo;
@com.alibaba.fastjson.annotation.JSONField(name = "poiweight")
private List<?> poiweight;
@com.alibaba.fastjson.annotation.JSONField(name = "distance")
private String distance;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_ext")
private BizExtDTO bizExt;
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
private List<PhotosDTO> photos;
@lombok.NoArgsConstructor
@lombok.Data
public static class BizExtDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "rating")
private List<?> rating;
@com.alibaba.fastjson.annotation.JSONField(name = "cost")
private String cost;
}
@lombok.NoArgsConstructor
@lombok.Data
public static class PhotosDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "title")
private List<?> title;
@com.alibaba.fastjson.annotation.JSONField(name = "url")
private String url;
}
}