Merge branch 'script_gj' into 'master'

Script gj

See merge request !1193
This commit is contained in:
2025-11-19 11:36:52 +08:00
12 changed files with 226 additions and 43 deletions
@@ -813,8 +813,9 @@ class CommonDatautils {
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","经纬天地园二期")
// q.put("name","扬州北街8号")
DBObject keys = new BasicDBObject();
if(fields){
@@ -211,6 +211,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def getFzDistricts(def city, def mdate, def endMDate){
def q = new BasicDBObject()
q.put("city", city)
if(mdate){
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
}
@@ -99,6 +99,7 @@ class DistrictPriceExportBatch extends BaseBatch{
def getFzDistricts(def city, def mdate, def endMDate){
def q = new BasicDBObject()
q.put("city", city)
if(mdate){
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
}
@@ -37,7 +37,7 @@ class DistrictSellStatBatch extends BaseBatch {
@Override
void process(Object args) {
def months = DateUtils.getMonthList(202209)
def months = DateUtils.getMonthList(202510)
months = months[0..(months.size()-1)]
println months
for(def month : months){
@@ -69,9 +69,9 @@ class DistrictSellStatBatch extends BaseBatch {
if(!curDate){
curDate = new Date()
}
def citys = CommonUtils.findAllCity()
citys.remove("上海")
citys.remove("福州")
def citys = ["上海"]//CommonUtils.findAllCity()
// citys.remove("上海")
// citys.remove("福州")
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(curDate))
@@ -89,9 +89,7 @@ class DistrictSellStatBatch extends BaseBatch {
for(def city : citys){
index++
if(index < 242){
continue
}
def districts = []
if(city == "福州"){
if(day == 29){
@@ -831,7 +831,7 @@ class OpenApiUtils {
static def addrVerifyGuangda(city, address){
address = URLEncoder.encode(address, "utf-8");
def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
// def url = "http://localhost:8080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
if(city){
url = url + "&city=${city}"
}
@@ -33,8 +33,8 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
task.addDev(path)
// task.addDevAddress(path)
// task.addDev(path)
task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -3,32 +3,160 @@ package com.fangjia.script.gj.district
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONNull
import org.bson.types.ObjectId
class DistrictLocationGeo {
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
static main(args) {
def task = new DistrictLocationGeo()
task.run()//调用高德逆解析,并保存数据
task.run()//通过经纬度,调用高德逆解析,并保存数据
// task.run2();//通过地址,调用高德逆解析
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\京东\\需逆解析数据.xlsx", { d ->
def jsonObject = GaoDeApiUtils.geocode(d['C'])
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
if(d['E']){
def jsonObject = GaoDeApiUtils.geocode(d['E'])
if(jsonObject){
d["F"] = "高德"
def dbo = new BasicDBObject()
dbo.put("old_id", d['A'])
dbo.put("city", d['B'])
dbo.put("region", d['C'])
dbo.put("name", d['D'])
dbo.put("old_location", d['E'])
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
// addrComp.remove("neighborhood")
// addrComp.remove("building")
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
println(dbo)
allSeaweedGaodePoisColl.save(dbo)
}
}
})
}
static BasicDBObject convertToDBObject(Map map) {
if (!map) return null
BasicDBObject dbObject = new BasicDBObject()
map.each { key, value ->
def convertedValue = convertValue(value)
if (convertedValue != null) {
dbObject.put(key, convertedValue)
}
}
return dbObject
}
static Object convertValue(Object value) {
if (value == null) return null
switch (value) {
case Map:
return convertToDBObject(value as Map)
case List:
return convertList(value as List)
default:
return value
}
}
static BasicDBList convertList(List list) {
BasicDBList dbList = new BasicDBList()
list.each { item ->
def convertedItem = convertValue(item)
if (convertedItem != null) {
dbList.add(convertedItem)
}
}
return dbList
}
static def cleanGeoData(data) {
if (data == null) return null
def result = [:]
data.each { key, value ->
def cleanedValue = cleanValue(value)
if (cleanedValue != null && !isEmptyCollection(cleanedValue)) {
result[key] = cleanedValue
}
}
return result
}
static def cleanValue(value) {
if (value == null) return null
switch (value) {
case Map:
def cleanedMap = [:]
value.each { k, v ->
def cleaned = cleanValue(v)
if (cleaned != null && !isEmptyCollection(cleaned)) {
cleanedMap[k] = cleaned
}
}
return cleanedMap ? cleanedMap : null
case List:
def cleanedList = value.collect { cleanValue(it) }
.findAll { it != null && !isEmptyCollection(it) }
return cleanedList ? cleanedList : null
case JSONNull:
return null
default:
// 检查是否是包含JSONNull的字符串表示
if (value instanceof String && value.contains("JSONNull")) {
return null
}
return value
}
}
static def isEmptyCollection(obj) {
if (obj instanceof Map) return obj.isEmpty()
if (obj instanceof List) return obj.isEmpty()
return false
}
def run2(){
ExcelUtils.modify("D:\\数据\\昆仑\\263未出居委数据.xlsx",{ d->
def jsonObject = GaoDeApiUtils.geocodeByAddress(d['C'], d['B'])
Thread.sleep(1000)
def count = 0
if (jsonObject) {
def dbo = new BasicDBObject()
dbo.put("old_id", d['B'])
dbo.put("old_location", d['C'])
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
dbo.put("addressComponent", jsonObject.regeocode.addressComponent)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
seaweedGaodePoisColl.save(dbo)
def rets = jsonObject.geocodes
if(rets){
def dbo = rets[0]
if(dbo){
d['D'] = dbo.formatted_address
d['E'] = dbo.level
}
}
count = jsonObject.regeocode.pois.size()
}
d['D'] = count
@@ -355,7 +355,7 @@ class FangjiaClientUtils {
address = address.toString().replace("{","")
int retryCount = 0
while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
url = "http://192.168.0.100:32080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -1,19 +1,36 @@
package com.fangjia.script.gj.village
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
class KunlunVillage {
static main(args) {
def task = new KunlunVillage();
task.run()
// task.addrCheck()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\昆仑\\3987户需批量更新涉农信息明细-20251023-结果.xlsx",{ d->
def addrCheck(){
if(!d['R'] && d['B']){
def dto = OpenApiUtils.addrVerifyGuangda(null, d['B'])
ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d->
def obj = FangjiaClientUtils.formatNinelevel(null, d["C"])
if(obj){
println obj
d['AB'] = obj.addressAssessment.msg
}
})
}
def run(){
def index = 0
ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d->
index++
if(!d['R'] && d['C']){
println index
def dto = OpenApiUtils.addrVerifyBase(null, d['C'])
if(dto){
// def dto = jsonObj.result
d['F'] = dto.province
@@ -56,7 +56,7 @@ class CZDataBatch extends BaseBatch{
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
def commids = findAllJnBankDatas(city)
//
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
FzDataUtils.removeNoUpdate(city, districts)
districtRun(districts, statMap, commids);
@@ -219,14 +219,14 @@ class CZDataBatch extends BaseBatch{
// 商铺名称
map.name = shop.name
map.alias = shop.alias
map.build_date = getBuildDate(shop);
map.build_date = FzDataUtils.getBuildDate(shop);
//附属属性
map.build_type = shop.build_type
map.property_life = shop.property_life
map.estate_type2 = shop.estate_type2
map.build_structure = shop.build_structure
// 楼盘均价
map.avg_price = getShopPrice(priceString, shop.city, shop.region, shop.name);
map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name);
// 省 市 区
if(shop.city == "常州"){
map.province_code = regionMap.get("江苏");
@@ -241,28 +241,28 @@ class CZDataBatch extends BaseBatch{
// 地址
map.address = shop.address
// 所属区域
map.in_region_code = getRegionCode(shop.region);
map.in_region_code = FzDataUtils.getRegionCode(shop.region);
// 商铺类型
map.estate_type = getShopEstateTypeCode(shop.estate_type2);
map.estate_type = FzDataUtils.getShopEstateTypeCode(shop.estate_type2);
// 公共服务配套
map.public_service_set_code = getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2);
map.public_service_set_code = FzDataUtils.getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2);
// 基础设施
map.facilities_code = "01";
// 商业密度
map.sophistication_code = getSophisticationCode(shop.city, shop.lat2, shop.lng2);
map.sophistication_code = FzDataUtils.getSophisticationCode(shop.city, shop.lat2, shop.lng2);
// 交通条件
map.traffic_code = getTrafficCode(shop.city, shop.lat2, shop.lng2);
map.traffic_code = FzDataUtils.getTrafficCode(shop.city, shop.lat2, shop.lng2);
// 泊车条件 /
map.stop_car_code = "01";
// 临街道路
map.street_code = getStreetCode(street);
map.street_code = FzDataUtils.getStreetCode(street);
// 租金
map.shop_rent = getRentNumber(rentPrice, shop.city, shop.region, shop.name);
map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name);
// 物业公司
map.estate_company = getEstateCompany(shop.estate_company);
map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company);
// 物业费(每月每平米)
map.estate_charge = getEstateCharge(shop.estate_charge);
map.estate_charge = FzDataUtils.getEstateCharge(shop.estate_charge);
saveShop(map);
}
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 6
def nThread = Runtime.getRuntime().availableProcessors() * 8
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
@@ -28,7 +28,10 @@ class GaoDeApiUtils {
params.put("location", location)
params.put("extensions","all")
params.put("radius", "3000")
params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")
// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")//´¿×¡Õ¬ÀàÐÍ
//¾©«²âÊÔÔÊÐíÀàÐÍ
params.put("poitype", "100000|100100|100101|100102|100103|100104|100105|100200|100201|120000|120200|120203|120300|120301|120302|120303|120304|190000|190100|190400|190401|190402|190403|190500|190600|190700")
String content
def stop = true
JSONObject jsonObj = null
@@ -56,6 +59,40 @@ class GaoDeApiUtils {
return jsonObj
}
def static geocodeByAddress(String city, String address) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("address", address)
params.put("batch", "true")
params.put("city", city)
// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")
String content
def stop = true
JSONObject jsonObj = null
def errorCount = 1
while (stop){
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8")
}
jsonObj = JSONObject.fromObject(content)
println jsonObj
if(jsonObj.status!="1"){
println jsonObj
errorCount ++
Thread.sleep(3000)
if(errorCount>3){
stop = false
}
}else{
stop = false
}
}
return jsonObj
}
def static geo(city, address) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", "a34894d151fffff94f9a33bae3d5565c")//ÐíéF½Ü