Merge branch 'script_gj' into 'master'

Script gj

See merge request !1179
This commit is contained in:
2025-10-28 14:11:26 +08:00
12 changed files with 292 additions and 43 deletions
@@ -28,7 +28,7 @@ class DistrictPriceExportBatch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村9-13号、15-18号":37500,"津泰新村14号":51000,"津泰新村1-6号、8号":23500,"津泰新村7号":38000,"津泰新村19号":36000]
def specialDistsPriceMap = ["津泰新村9-13号、15-18号":37500,"津泰新村14号":51000,"津泰新村1-6号、8号":23500,"津泰新村7号":38000,"津泰新村19号":36000, "金泉公寓15、16、17栋":24000,"金泉小区(1-12栋)":14000]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -20,7 +20,7 @@ class GongShu {
}
def run() {
String path1 = "D:\\数据\\福州\\福州200个小区抽样价格核验202509.xlsx";
String path1 = "D:\\数据\\福州\\福州200个小区抽样价格核验202510.xlsx";
// String path1 = "D:\\数据\\福州\\价格比对_20240915.xlsx";
String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
// String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
@@ -36,8 +36,8 @@ class GongShu {
// match(path2, path1)
// match2(path3, path1)
// houseCase(path2)
thirdPrice(path1)
// priceAvg(path1)
// thirdPrice(path1)
priceAvg(path1)
// futurePrice(path2)
// rePrice(path2)
}
@@ -329,7 +329,7 @@ class GongShu {
def city = "福州"
def region = d['B']
def name = d['C']
def valusMap = CommonDatautils.findThirdPartyPrice(city, region, name, 202505, 202508)
def valusMap = CommonDatautils.findThirdPartyPrice(city, region, name, 202506, 202509)
if (valusMap) {
def dataMap = valusMap.get(city + "##" + region + "##" + name)
if (dataMap.get("anjuke_esf_price")) {
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202508.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202509.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -635,8 +635,25 @@ class OpenApiUtils {
}
static def similarRank(proCode, cityCode, regionCode, name, area, floor, height){
def url = "http://localhost:8080/fangjia-open/house/similar?aboveFloorAmount=${height}&buildingName=${name}&city=${cityCode}&district=${regionCode}&" +
"propertyTypeCode=01&province=${proCode}&floorNumber=${floor}&strctureArea=${area}"
def url = "http://open.fangjia.com/house/similar?aboveFloorAmount=${height}&buildingName=${name}&city=${cityCode}&district=${regionCode}&" +
"propertyTypeCode=01&province=${proCode}&floorNumber=${floor}&strctureArea=${area}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
static def shopSimilarRank(proCode, cityCode, regionCode, name, area, floor, height){
def url = "http://open.fangjia.com/house/shopSimilar?aboveFloorAmount=${height}&buildingName=${name}&city=${cityCode}&district=${regionCode}&" +
"province=${proCode}&floorNumber=${floor}&strctureArea=${area}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
@@ -713,7 +730,7 @@ class OpenApiUtils {
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
print(url)
println(url)
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
@@ -725,6 +742,21 @@ class OpenApiUtils {
}
static def propertyRating(id){
def url = "http://open.fangjia.com/address/propertyRating?id=${id}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
// print(url)
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
//源地址的完整性进行验证
static def addrVerify(city, address){
def url = "http://open.fangjia.com/addrSvc/addrVerify?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
@@ -781,7 +813,7 @@ class OpenApiUtils {
static def addrVerifyGuangda(city, address){
address = URLEncoder.encode(address, "utf-8");
def url = "http://192.168.0.19:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(city){
url = url + "&city=${city}"
@@ -798,6 +830,24 @@ class OpenApiUtils {
}
static def addrVerifyBase(city, address){
address = URLEncoder.encode(address, "utf-8");
def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(city){
url = url + "&city=${city}"
}
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.msgCode ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
// 省、市、区、街道、社区居委、POI补全
static def complete(city, address){
def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
@@ -3,14 +3,18 @@ package com.fangjia.script.gj.abc
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.CopyOnWriteArrayList
@@ -24,11 +28,15 @@ class SuzouPocTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
public static DBCollection COMMUNITY_RATE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "community_rate", "mongo.db.fangjia.com")
static main(args) {
def task = new SuzouPocTest()
// task.run()
// task.newRun()
task.test()
// task.test()//有楼层
// task.testEval()//无楼层
task.level()//楼盘评级
}
def test(){
@@ -47,6 +55,78 @@ class SuzouPocTest {
}
def level(){
ExcelUtils.modify("D:\\数据\\其它\\附件1:POC测试样本(匹配).xlsx", { d ->
if(d['O']){
// def obj = CommonDatautils.findDistrict(d['K'], d['L'], d['M'], "district")
// if(obj){
// d['N'] = obj._id.toString()
// def ret = OpenApiUtils.propertyRating(obj._id.toString())
// def ret = OpenApiUtils.financialRating(d['K'], d['K']+d['L']+d['M'])
def ret = findCommunityRate(d['O'])
if(ret){
d['X'] = ret.community_price_rating
// d['W'] = ret.total_score
}
// def summary = findCommunitySummary(obj._id.toString())
// if(summary){
// d['O'] = summary.building_rating
// }
// }
}
})
}
def findCommunityRate(id){
def q = new BasicDBObject()
q.put("date", 202509)
q.put("comm_id", id)
return COMMUNITY_RATE_COLL.findOne(q)
}
def findCommunitySummary(id){
def obj = null
def sql = "select * from community_summary where id='${id}'"
ElasticUtils.executeSqlQuery(sql, {
obj = it
})
return obj
}
def testEval(){
ExcelUtils.modify("D:\\数据\\其它\\附件1:POC测试样本(匹配).xlsx", { d ->
if(d['O'] && !d['T']){
// def city = d['C']
// if(city == "市辖区"){
// city = d['B']
// }
def obj = CommonDatautils.findDistrictById(d['O'])
if(obj){
d['R'] = obj.city
d['S'] = obj.region
d['T'] = obj.name
def ret1 = OpenApiClient.addressEvaluation(obj.city, obj.region, obj.city+obj.region+obj.name, obj.name, d['F'], null, null)
if (ret1) {
d['P'] = ret1.totalPrice
d['Q'] = ret1.avgPrice
}else{
def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name)
if(dbo) {
d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble())
d['Q'] = dbo.avg_price_2b
}
}
}
}
});
}
def newRun(){
println("start:"+new Date().toString())
@@ -189,7 +189,7 @@ class AddDevShapeData {
def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\0903'
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\1020'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
dir.eachFileRecurse(FileType.FILES) { file ->
@@ -31,48 +31,85 @@ class AutoAddDistrictBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fullCityMap = null//CommonDatautils.queryFullNameCity()
def mcaDistMap = null//CommonDatautils.queryDistNameCityByMapping()
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
// runBatch(new AutoAddDistrictBatch(), args)
runBatch(new AutoAddDistrictBatch(), args)
def task = new AutoAddDistrictBatch();
// def task = new AutoAddDistrictBatch();
// task.checkMap()
task.addDistrict()
// task.addDistrict()
}
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = ["泉州"]//CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
for(def c : citys){
def noMatchDatas = findNoMatchResult(c)
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
def rets = checkDatas(noMatchDatas.keySet())
list.addAll(rets)
}
// def citys = ["苏州"]//CommonDatautils.findAllCity()
// def list = []
// list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型", "高德名称", "库中是否存在", "是否新增入库"]
// def citystr = ""
// for(def c : citys){
// def noMatchDatas = findNoMatchResult(c)
// println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
// def rets = checkDatas(noMatchDatas.keySet())
// list.addAll(rets)
// citystr = citystr + c+"_"
// }
//
// def fileName = "/home/goojia/gj/add_dist_info.xlsx"
// if (CommonDatautils.isWindow()) {
// fileName = "D:\\数据\\数据建设\\小区\\"+citystr+"待新增小区.xlsx"
// }
// ExcelUtils.write(fileName, list, "待入库数据")
//
// check(fileName)
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\泉州待新增小区.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
check(fileName)
addFlag("D:\\数据\\数据建设\\小区\\苏州_待新增小区.xlsx")
}
def addFlag(fileName){
ExcelUtils.modify(fileName,{d->
if(!d['L'] && d['I'].toString().contains('住宅')){
if(d['K']){
d['L'] = "不新增"
}else if(d['C'].toString().endsWith("路") || d['C'].toString().endsWith("街") || d['C'].toString().endsWith("广场") ){
d['L'] = "不新增"
}else if(d['C'].toString().contains("公共租赁") || d['C'].toString().contains("自建房") || d['C'].toString().contains("商场")){
d['L'] = "不新增"
}else if(d['C'].toString().endsWith("路小区") && d['E'].toString().toDouble().toInteger()==0){
d['L'] = "不新增"
}else if(d['C'].toString().contains("社区") && d['E'].toString().toDouble().toInteger()==0){
d['L'] = "不新增"
}else if(d['B'].toString().contains("周边")){
d['L'] = "不新增"
}else if((d['C'].toString().endsWith("村") || d['C'].toString().endsWith("巷") || d['C'].toString().contains("街小区")) && d['F'].toString().toDouble().toInteger()==0){
d['L'] = "不新增"
}else if((d['C'].toString().endsWith("号小区") || d['C'].toString().endsWith("宿舍"))&& d['E'].toString().toDouble().toInteger()==0 && d['F'].toString().toDouble().toInteger()==0){
d['L'] = "不新增"
}else if(d['C'].toString().contains("路") && d['C'].toString().contains("号") && d['E'].toString().toDouble().toInteger()==0){
d['L'] = "不新增"
}else if(d['G'] && (d['G'].toString().contains("行政地标") || d['G'].toString().contains("旅游") ||d['G'].toString().contains("休闲")||d['G'].toString().contains("购物")||d['G'].toString().contains("生活服务") )){
d['L'] = "不新增"
}else if(d['J'] && d['C']==d['J'] && d['I'].toString().contains('住宅')){
d['L'] = "新增"
}else if (d['G'] && d['G'].contains("住宅")){
d['L'] = "新增"
}
}
})
}
def check(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def dists = CommonDatautils.findAllDistrictByName(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中已存在"
d['K'] = "库中已存在"
}else{
dists = CommonDatautils.findDistrictByAlias(d[0], d[1], d[2], "district")
if(dists){
d['J'] = "库中别名中存在"
d['K'] = "库中别名中存在"
}
}
})
@@ -192,7 +229,7 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("match_id", new BasicDBObject('$exists',false))
q.put("old_match_id", new BasicDBObject('$exists',false))
q.put("city", c)
q.put("district", "丰泽")
// q.put("district", "丰泽")
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
def rMap = [:]
for(def d : dbcursor){
@@ -33,8 +33,8 @@ class DevUpdateTestBatch {
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
task.addDevAddress(path)
task.addDev(path)
// task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -26,6 +26,7 @@ class ApiTest {
def task = new ApiTest()
// task.rankRunTest()//小区排名接口测试
// task.similarRunTest()//相似房源接口测试
task.jnBankSearch() //江南行小区搜索
// task.loupanLevelTest()//楼盘评级接口测试
// task.jsAbcHouseCastTest()//北交所房源接口测试 小区
@@ -33,7 +34,7 @@ class ApiTest {
// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价
task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
// task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
// task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
@@ -206,6 +207,12 @@ class ApiTest {
})
}
def jnBankSearch(){
ExcelUtils.modify("D:\\数据\\江南农商行\\住宅信息V2-差异案例(反馈).xlsx", { d ->
})
}
def similarRunTest(){
ExcelUtils.modify("D:\\数据\\江南农商行\\住宅信息V2-差异案例(反馈).xlsx", { d ->
@@ -10,9 +10,9 @@ class GuangDaVillage {
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\光大\\公开数据信息补充v3.xlsx",{ d->
ExcelUtils.multiThreadingModify("D:\\数据\\光大\\公开数据地址情况.xlsx",{ d->
if(!d['R']){
if(!d['R'] && d['A']){
def dto = OpenApiUtils.addrVerifyGuangda(null, d['A'])
if(dto){
// def dto = jsonObj.result
@@ -1,7 +1,11 @@
package com.fangjia.script.jiangnanblank
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -13,9 +17,80 @@ class TestEvaluation {
static main(args) {
def task = new TestEvaluation();
task.run();
// task.run();
// task.match()
task.houseSimilar()
}
def houseSimilar(){
ExcelUtils.modify("D:\\数据\\江南农商行\\异地押品未自动重估明细(小区与无样本).xlsx",{d->
if(d['I'] && d['C']=="商铺、商场"){
def dist = CommonDatautils.findDistrictById(d['I'])
if(dist){
def map = CcbCommonUtils.findAllDistrictCodeValue(dist.city)
if(map){
def codeObj = map.get(dist.region)
if(codeObj){
def cityCode = codeObj.city_code
def regionCode = codeObj.district_code
def provCode = codeObj.city_code.toString().substring(0,2)+"0000"
def height = null
if(d['M']){
height = d['M'].toString().toDouble().toInteger()
}
def floor = null
if(d['N']){
floor = d['N'].toString().toDouble().toInteger()
}
// def results = com.fangjia.script.gj.OpenApiUtils.similarRank(provCode, cityCode,regionCode, d['L'], d['O'], floor, height)
def results = com.fangjia.script.gj.OpenApiUtils.shopSimilarRank(provCode, cityCode,regionCode, d['L'], d['O'], floor, height)
if(results){
def str = ""
for(def r : results){
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"\n"
}
d['Q'] = str
}
}
}
}
}
})
}
def match(){
ExcelUtils.modify("D:\\数据\\江南农商行\\异地押品未自动重估明细(小区与无样本).xlsx",{d->
if(["商铺、商场"].contains(d['C'])){
if(d['H'] && !d['I']){
def address = d['H']
def city = d['E']
if(city=="市辖区"){
city = d['D']
}
if(!address.contains("省") && !address.contains("市")){
address = d['D'] +city + d['F']+d['H']
}
def jsonObject = FangjiaClientUtils.newMatch(city, address, "shop", false)
if(!jsonObject && d['G']){
jsonObject = FangjiaClientUtils.newMatch(city, d['F']+d['G'], "shop", false)
}
if(jsonObject){
d['I'] = jsonObject.get("id")
d['J'] = jsonObject.get("city")
d['K'] = jsonObject.get("district")
d['L'] = jsonObject.get("name")
}
}
}
})
}
def run(){
def citys = CommonDatautils.findAllCity();
for(def city :citys){
@@ -132,7 +132,7 @@ public class CitySellStatServiceImpl implements CitySellStatService {
q.add(new QueryParam("date", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
List<SortParam> s = new ArrayList<SortParam>();
s.add(new SortParam("date", false));
s.add(new SortParam("date", true));
return citySellStatDao.findAllByParams(q, s, count);
}