Merge remote-tracking branch 'origin/master'

This commit is contained in:
2025-07-11 14:13:24 +08:00
8 changed files with 311 additions and 96 deletions
+19
View File
@@ -555,6 +555,25 @@
<version>0.0.12</version>
</dependency>
<dependency>
<groupId>org.codehaus.groovy.modules.http-builder</groupId>
<artifactId>http-builder</artifactId>
<version>0.7.1</version>
<exclusions>
<!-- 排除冲突的 xmlbeans -->
<exclusion>
<groupId>xmlbeans</groupId>
<artifactId>xmlbeans</artifactId>
</exclusion>
<!-- 如果还有其他冲突的依赖(如 xercesImpl),也一并排除 -->
<exclusion>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
@@ -443,7 +443,7 @@ class CcbCommonUtils {
}
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","新泾三村")
// q.put("name","龙源星城")
// q.put("name_cat", 0);
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
@@ -105,6 +105,8 @@ class CommonDatautils {
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
public static final DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -811,6 +813,8 @@ class CommonDatautils {
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","经纬天地园二期")
DBObject keys = new BasicDBObject();
if(fields){
@@ -4142,9 +4146,13 @@ class CommonDatautils {
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range)
if(region){
queryBuilder.must(QueryBuilders.termQuery("region", region))
}
if(name || alias){
if(!alias){
alias = []
@@ -4250,6 +4258,19 @@ class CommonDatautils {
return rets;
}
static def findAllNewPoiDatas(city, region, name, small_type){
def rets = []
def index= 0
String sql = "select cityname,adname,name,address,location,type,id,parent from amap_poi_2024 where cityname='${city}'"
ElasticUtils.executeSqlQuery(sql, { map->
// if(map.address && map.address.contains("五角场")){
index++
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y, "parent_id":map.parent]
// }
})
return rets;
}
@@ -5644,4 +5665,101 @@ class CommonDatautils {
})
return rets;
}
static def findGaoDeDatas(city, region, name){
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') limit 100"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name){ //&& d.shape
obj = d
}
})
return obj;
}
static def findGaoDeDatas2(city, region, name){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
def body = """
{
"_source": {
"includes": ["cat", "address", "id","name", "township","location"]
},
"query": {
"bool": {
"must": [
{
"term": {
"city" : "${city}"
}
},
{
"term": {
"district" : "${region}"
}
},
{
"match": {
"name": {"query": "${name}","operator": "and"}
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
// if(name=="河北河间经济开发区"){
// println name
// }
def esObj = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
def d = hits[0]
// if(d._source.cat.contains("住宅")){
def source = d._source
def newName = source.name
if(newName == name){
esObj = source
}else if (newName.contains("开发区") && name.contains("开发区") && (newName.contains(name) || name.contains(newName))){
esObj = source
}else if(newName.contains("园区") && name.contains("园区") && (newName.contains(name) || name.contains(newName))){
esObj = source
}else if(newName.contains("示范区") && name.contains("示范区") && (newName.contains(name) || name.contains(newName))){
esObj = source
}else{
if(name.contains("开发区")){
name = name.replace("国家高新技术产业开发区","").replace("高新技术产业开发区","").replace("经济技术开发区","").replace("经济开发区", "")
newName = newName.replace("国家高新技术产业开发区","").replace("高新技术产业开发区","").replace("经济技术开发区","").replace("经济开发区", "")
}
if(newName == name){
esObj = source
}
}
// }
}
}
return esObj
}
static def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def dbcursor = BAIDU_POI_COLL.find(q)
def address = null
for(def d : dbcursor){
address = d.addr
}
return address;
}
}
@@ -189,7 +189,7 @@ class AddDevShapeData {
def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\临时-没付费\\'
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\623西安'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
dir.eachFileRecurse(FileType.FILES) { file ->
@@ -203,7 +203,7 @@ class AddDevShapeData {
def addDevInfo(file){
ExcelUtils.multiThreadingModify(file, {d->
// if(!d['I']){
if(!d['I']){
def name = d['E']
def q = new BasicDBObject()
q.put('_id',new ObjectId(d['A']))
@@ -218,24 +218,26 @@ class AddDevShapeData {
if(obj.shape){
d['T'] = "有"
}
// if(obj.m_date){
// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
// }
if(obj.m_date){
d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd")
}
// if(devAreaMap){
// devAreaMap.each {k,v->
// if(obj.name && k && obj.name.contains(k)){
// d['M'] = v.area_info
// d['N'] = v.count
// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
// }
// }
//
// }else{
// println name+" 空值"
// }
if(devAreaMap){
devAreaMap.each {k,v->
if(obj.name && k && obj.name.contains(k)){
d['M'] = v.area_info
d['N'] = v.count
d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
}
}
}else{
println name+" 空值"
}
}else{
d['S'] = "删除"
}
// }
}
})
}
@@ -269,13 +271,15 @@ class AddDevShapeData {
DEVELOPMENT_ZONE_43_COLL.save(obj)
}else{
udata.put("shape", resultMap)
// udata.put("m_date", new Date())
udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
udata.put("operate_result" , "已处理")
udata.put("operator", "外包")
DEVELOPMENT_ZONE_43_COLL.update(q, new BasicDBObject('$set', udata), false, false)
}
}
}else{
println "【开发区】"+retMap.name +" 库中未查到"
}
}
}
@@ -296,7 +300,7 @@ class AddDevShapeData {
areaMap.count = coordinates.size()
def areas = []
// if(name.contains("江苏常熟综合保税区")){
if(name.contains("陕西蓝田工业园")){
for(def pointList : coordinates){
index ++
def gaode_coordinates = []
@@ -339,7 +343,7 @@ class AddDevShapeData {
}
areaMap.area_info = areas.join("\n")
// }
}
}else{
println file +" 有多个"
@@ -31,16 +31,16 @@ class AutoAddDistrictBatch extends BaseBatch{
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
// runBatch(new AutoAddDistrictBatch(), args)
runBatch(new AutoAddDistrictBatch(), args)
def task = new AutoAddDistrictBatch();
task.checkMap()
// def task = new AutoAddDistrictBatch();
// task.checkMap()
}
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = CommonDatautils.findAllCity()
def citys = ["三明"]//CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
for(def c : citys){
@@ -52,7 +52,7 @@ class AutoAddDistrictBatch extends BaseBatch{
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\add_dist_info_1.xlsx"
fileName = "D:\\数据\\数据建设\\小区\\add_dist_info_1.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
@@ -102,12 +102,14 @@ class AutoAddDistrictBatch extends BaseBatch{
def gaode_obj = findGaodeCount(arrs[0], arrs[1], arrs[2])
def gaode_count = 0
def gaode_type = null
def gaode_name = ""
if(gaode_obj){
gaode_count = 1
gaode_type = gaode_obj.type
gaode_name = gaode_obj.name
}
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type]
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type, gaode_name]
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -165,7 +167,7 @@ class AutoAddDistrictBatch extends BaseBatch{
String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name){ //&& d.shape
if(d.name == name || d.name.contains(name) || name.contains(d.name)){ //&& d.shape
obj = d
}
})
@@ -1,11 +1,13 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -44,6 +46,7 @@ class DevProcess {
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
// task.findDev()
// task.calAddrDistince()
// task.updateCityAndDist()
@@ -57,7 +60,7 @@ class DevProcess {
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
task.synchDevTo100()
// task.synchDevTo100()
// task.statShape()
// task.addNewDev2()
@@ -85,9 +88,19 @@ class DevProcess {
// task.updateDevShape()//修改开发区结构
task.aiDevTown()//ai获取结果
}
def aiDevTown(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
def message = res.data.replaceAll("\n", "")
d['F'] = res.code
d['G'] = message
println d['E'] +" "+res.code + " "+message
})
}
def parseCityAndDist(){
@@ -440,35 +453,80 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\高新区.xls",{d->
if(!d['E'] && d['D']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\2018年版.xlsx",{d->
if(!d['G'] && d['C']){
def q = new BasicDBObject()
// q.put("name", d['D'])
q.put("city",d['C'])
// q.put('alias', new BasicDBObject('$in',[d['D']]))
def name = d['D'] //.replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put("name", d['C'])
def way = "相同"
def obj = development_zone_CONN.findOne(q)
if(obj){
d['G'] = "相似"
// d['K'] = obj.name
// d['C'] = obj._id.toString()
d['E'] = obj.name
// if(obj.name == d['D']){
// d['G'] = "相同"
// }else{
// d['G'] = "包含"
// }
if(obj.shape){
d['F'] = "是"
}
// d['E'] = "别名"
// d['F'] = obj.name
if(!obj){
q.put('alias', new BasicDBObject('$in',[d['C']]))
obj = development_zone_CONN.findOne(q)
way = "别名"
}
if(!obj){
def name = d['C'] //.replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
obj = development_zone_CONN.findOne(q)
way = "模糊"
}
def allDevNames = []
if(!obj){
def query = new BasicDBObject()
def keys = new BasicDBObject()
keys.put("name", 1)
keys.put("alias", 1)
allDevNames = development_zone_CONN.find(query, keys).toList()
}
if(obj){
d['H'] = obj.name
d['G'] = way
}else{
if(allDevNames){
def maxScore = -100
def smilarObj = null
for(def dev : allDevNames){
def devNames = []
if(dev.alias){
devNames = dev.alias
}
devNames << dev.name
for(def aliasName : devNames){
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(d['C'], aliasName);
if(nameScore>maxScore){
maxScore = nameScore
smilarObj = dev
}
}
}
if(smilarObj){
d['H'] = smilarObj.name
d['G'] = "相似得分"
d['I'] = maxScore
}
}
}
}
},0);
}
def findDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区外包\\开发区-0423(161个-浙江).xlsx", { d ->
def q = new BasicDBObject()
q.put("name", d['B'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['C'] = obj.city
d['D'] = obj.district
}
})
}
def calAddrDistince(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
if(d['M'] && d['N'] && d['Y'] && d['Z']){
@@ -5,6 +5,8 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import com.mongodb.BasicDBObject
import groovyx.net.http.ContentType
import groovyx.net.http.RESTClient
import net.sf.json.JSONArray
import net.sf.json.JSONObject
@@ -423,25 +425,17 @@ class FangjiaClientUtils {
}
static def aiResult(content){
def headers = [:]
headers.put("accept", "application/json")
headers.put("Content-Type", "application/json")
def url = "http://didata-cloud-server.fangjia.com/chat/deepseek"
try {
Map<String, String> params = new HashMap<String, String>();
params.put("content", content);
String result = HttpClient.post(url, params, headers, "utf-8", true)
println result
if (result) {
def json = JSONObject.fromObject(result)
return json
}
} catch (Exception e) {
println e.getMessage()
}
return null
def client = new RESTClient('http://didata-cloud-server.fangjia.com/')
def response = client.post(
path: '/chat/deepseek',
contentType: ContentType.JSON,
headers: [accept: 'application/json'],
body: [content: content]
)
//
// println "Response status: ${response.responseData}"
// println "Response data: ${response.data}"
return response.data
}
}
@@ -8,25 +8,41 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 浙商本地化部署同步脚本-逻辑修改
* 浙商改为接口,所以不用以前的数据
*/
@BatchOwner(Watcher.GaoJuan)
class DistrictLevelBatch extends BaseBatch{
public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
// public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
// public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL)
public static final zsDistLevelMap = [:]
public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new DistrictLevelBatch(), args);
}
@Override
public void process(Object args) {
def keys = new BasicDBObject("comm_id", 1)
keys.put("city", 1)
keys.put("region", 1)
keys.put("district_name", 1)
keys.put("status", 1)
keys.put("avg_price", 1)
keys.put("value_rate", 1)
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject(),keys)
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject())
for(def distLevel : zsDistLevelCursor){
zsDistLevelMap.put(distLevel._id.toString(), distLevel)
zsDistLevelMap.put(distLevel.comm_id.toString(), distLevel)
}
def citys = CommonDatautils.findAllCity();
@@ -44,6 +60,7 @@ class DistrictLevelBatch extends BaseBatch{
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
@@ -100,9 +117,11 @@ class DistrictLevelBatch extends BaseBatch{
def saveOrUpdateDB(city, datas){
def zsDistLevels = []
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city))
def zsDistLevelCursor = ZS_LEVEL_CONN.find(new BasicDBObject("city",city), new BasicDBObject("comm_id", 1))
for(def distLevel : zsDistLevelCursor){
zsDistLevels << distLevel._id.toString()
if(distLevel.comm_id){
zsDistLevels << distLevel.comm_id.toString()
}
}
def newDataMap = [:]
@@ -113,17 +132,17 @@ class DistrictLevelBatch extends BaseBatch{
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", d.region)
udata.put("name", d.name)
udata.put("level", d.level)
udata.put("avg_price", d.avg_price)
udata.put("district_name", d.name)
udata.put("value_rate", d.level)
// udata.put("avg_price", d.avg_price)
udata.put("status",0)
udata.put("update_date",new Date().getTime())
if(zsDistLevel){
if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.name!=d.name||zsDistLevel.level!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
if(zsDistLevel.city!=d.city||zsDistLevel.region!=d.region||zsDistLevel.district_name!=d.name||zsDistLevel.value_rate!=d.level||zsDistLevel.avg_price!=d.avg_price||zsDistLevel.status!=0){
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zsDistLevel._id), new BasicDBObject('$set',udata))
}
}else{
udata.put("_id", d._id.toString())
udata.put("comm_id", d._id.toString())
addDatas << new BasicDBObject(udata)
if(addDatas.size()%10000==0){
ZS_LEVEL_CONN.insert(addDatas);
@@ -134,16 +153,17 @@ class DistrictLevelBatch extends BaseBatch{
if(addDatas.size()>0){
ZS_LEVEL_CONN.insert(addDatas);
}
if(zsDistLevels.size()>0){
for(def zs_id : zsDistLevels){
def newData = newDataMap.get(zs_id.toString());
if(!newData){
def udata = new BasicDBObject()
udata.put("status",1)
udata.put("update_date",new Date().getTime())
ZS_LEVEL_CONN.update(new BasicDBObject("_id",zs_id), new BasicDBObject('$set',udata))
}
}
}
//先不删除,有些没价格
// if(zsDistLevels.size()>0){
// for(def zs_id : zsDistLevels){
// def newData = newDataMap.get(zs_id.toString());
// if(!newData){
// def udata = new BasicDBObject()
// udata.put("status",1)
// udata.put("update_date",new Date().getTime())
// ZS_LEVEL_CONN.update(new BasicDBObject("comm_id",zs_id), new BasicDBObject('$set',udata))
// }
// }
// }
}
}