Merge branch 'script_gj' into 'master'

update

See merge request !1238
This commit is contained in:
2026-02-24 16:37:15 +08:00
11 changed files with 395 additions and 37 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳10月份成交数据11.20.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳12月成交数据.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -122,7 +122,7 @@ class DistrictPriceExportBatch extends BaseBatch{
if(CommonDatautils.isWindow()){
districts = getFzDistricts(city, null, null)
} else{
if(day == 15){
if(day == 13){
districts = getFzDistricts(city, null, null)
}else{
districts = getFzDistricts(city, startMDate, endMDate)
@@ -20,7 +20,7 @@ class GongShu {
}
def run() {
String path1 = "D:\\数据\\福州\\福州200个小区抽样价格核验202601.xlsx";
String path1 = "D:\\数据\\福州\\福州200个小区抽样价格核验202602.xlsx";
// String path1 = "D:\\数据\\福州\\价格比对_20240915.xlsx";
String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
// String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
@@ -329,7 +329,7 @@ class GongShu {
def city = "福州"
def region = d['B']
def name = d['C']
def valusMap = CommonDatautils.findThirdPartyPrice(city, region, name, 202509, 202511)
def valusMap = CommonDatautils.findThirdPartyPrice(city, region, name, 202511, 202601)
if (valusMap) {
def dataMap = valusMap.get(city + "##" + region + "##" + name)
if (dataMap.get("anjuke_esf_price")) {
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202512.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202601.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -13,12 +13,13 @@ class DistrictLocationGeo {
final DBCollection JD_SEAWEED_COLL = MongoUtils.getCollectionByName("business", "jingdong_community", "192.168.0.206",27020)
final DBCollection JD_GEO_COLL = MongoUtils.getCollectionByName("business", "jingdong_geo", "192.168.0.206",27020)
final DBCollection JD_review_GEO_COLL = MongoUtils.getCollectionByName("business", "jingdong_review_geo", "192.168.0.206",27020)
final DBCollection JD_review_GEO_COLL = MongoUtils.getCollectionByName("business", "jingdong_update_location_geo", "192.168.0.206",27020)
static main(args) {
def task = new DistrictLocationGeo()
task.run()//通过经纬度,调用高德逆解析,并保存数据
// task.run()//通过经纬度,调用高德逆解析,并保存数据
// def distance = GeoUtils.calculateDistance(130.573501, 45.752712, 130.573527,45.753287);
// println distance
task.run1()
}
def run(){
@@ -53,9 +54,40 @@ class DistrictLocationGeo {
}
}
})
}
def run1(){
ExcelUtils.modify("D:\\数据\\京东\\经纬度更新5.3万.xlsx",{ d->
if(d['A']){
def lng = d['B'];
def lat = d['C'];
def jsonObject = GaoDeApiUtils.geocode(lng+","+lat)
if(jsonObject){
def obj = CommonDatautils.findDistrictById(d['A'])
d["F"] = "高德"
def dbo = new BasicDBObject()
dbo.put("city", obj.city)
dbo.put("district", obj.region)
dbo.put("name", obj.name)
dbo.put("comm_id", d['A'])
dbo.put("lng", lng)
dbo.put("lat", lat)
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
JD_review_GEO_COLL.save(dbo)
}
}
})
}
static BasicDBObject convertToDBObject(Map map) {
@@ -31,7 +31,7 @@ class FjwBigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
def sourceMap = ["冰山成交":"bingshanai.com","贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com",]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
@@ -0,0 +1,306 @@
package com.fangjia.script.gj.hongxing
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class HongXingMonthBatch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
static DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
static DBCollection HONGXING_MONTH_STAT = MongoUtils.getCollectionByName("business", "hongxing_month_stat", "mongo.db.fangjia.com")
static DBCollection SHOP_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "shop_sell_stat", "mongo.db.fangjia.com")
static DBCollection OFFICE_RENT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "office_rent_stat", "mongo.db.fangjia.com")
static DBCollection OFFICE_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "office_sell_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new HongXingMonthBatch(), args);
}
@Override
public void process(Object args) {
def lastMonthStr = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD(DateUtils.addMonth(new Date(), -1)),"yyyyMM");
def count = findCurMonthPrice(lastMonthStr)
if(count > 0){
def fileName = "/home/goojia/hongxing/202602.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\宏信息\\202602.xlsx"
}
def datas = ExcelUtils.read("D:\\数据\\宏信息\\最新0123核价结果(绿色价格)宏信估值详情(含总价).xlsx")
def newDatas = createDatas(datas, lastMonthStr)
def curDateStr = DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ExcelUtils.write("D:\\数据\\宏信息\\宏信楼盘监测"+curDateStr+".xlsx", newDatas,"结果")
}
}
def createDatas(datas, lastMonthStr){
def rows = []
rows << ["物业类型","名称","地址","面积","单价(元)","总价(元)","环比涨跌幅","定基涨跌幅(相比首月波动)","区域均价","区域均价涨跌幅","楼盘所在区域价格的排名(仅针对住宅)","楼盘所在区域排名变化(仅针对住宅)","期房/现房","省份","城市","城市排名"]
def index = 1
for(def d : datas){
println d['B']
index++
// if(index < 12){
// continue
// }
def row = []
row << d['D']
row << d['A']
row << d['B']
row << d['H']
if(!["商铺","办公楼"].contains(d['D'])){
def dist = CommonDatautils.findDistrictById(d['I'])
if(dist){
def priceObj = findDistrictMonthPrice(dist.city, dist.region, dist.name, lastMonthStr)
def tihuanMap = [:]
if(priceObj){
def markup = priceObj.mom
def curPrice = d['U']
def newAvgPrice = curPrice * markup + curPrice
row << DataUtil.getRound(newAvgPrice)
def totalPrice = newAvgPrice * d['H']
row << DataUtil.getRound(totalPrice)
row << markup*100
def djMarkup = (newAvgPrice - d['U'])*100/ d['U']
djMarkup = DataUtil.getRound(2, djMarkup)
row << djMarkup.toString()
tihuanMap.put(dist.city+"##"+dist.region+"##"+dist.name, newAvgPrice)
}else{
row << ""
row << ""
row << ""
row << ""
}
def regionPriceObj = findRegionPrice(dist.city, dist.region, lastMonthStr)
if(regionPriceObj){
row << regionPriceObj.avg_price
row << regionPriceObj.mom*100
}else{
row << ""
row << ""
}
def resMap = getDistSortNumber(dist.city, dist.region, lastMonthStr, tihuanMap)
def lastRank = null//getLastMonthSortNumer(d['AB'])
if(resMap){
row << resMap.num+"/"+resMap.total
if(lastRank){
row << lastRank - resMap.num
}else{
row << ""
}
}else{
row << ""
row << ""
}
}else{
println d['I'] +" 未找到该id的楼盘"
}
}else{
def dist = CommonDatautils.findDistrictById(d['I'])
def cityMarkupObj = null
def regionMarkupObj = null
if(dist){
if(d['D'] == "办公楼"){
cityMarkupObj = findRentOfficeMarkup(dist.city, null, lastMonthStr, "city")
regionMarkupObj = findSellOfficeMarkup(dist.city, dist.region, lastMonthStr, "district")
if(cityMarkupObj){
cityMarkupObj.mom = cityMarkupObj.mom / 100
}
if(regionMarkupObj){
regionMarkupObj.mom = regionMarkupObj.mom / 100
}
}else if(d['D'] == "商铺"){
cityMarkupObj = findSellShopMarkup(dist.city, null, lastMonthStr, "city")
regionMarkupObj = findSellShopMarkup(dist.city, dist.region, lastMonthStr, "district")
}
if(cityMarkupObj){
def newAvgPrice = d['U'] * cityMarkupObj.mom + d['U']
row << DataUtil.getRound(newAvgPrice)
def totalPrice = newAvgPrice * d['H']
row << DataUtil.getRound(totalPrice)
row << cityMarkupObj.mom * 100
def djMarkup = (newAvgPrice - d['U'])*100/ d['U']
djMarkup = DataUtil.getRound(2, djMarkup)
row << djMarkup.toString()
}else{
row << ""
row << ""
row << ""
row << ""
}
if(d['D'] == "办公楼"){
if(regionMarkupObj){
row << (int)regionMarkupObj.avg_price
row << regionMarkupObj.mom * 100
}else{
row << ""
row << ""
}
}else{
if(regionMarkupObj){
row << (int)regionMarkupObj.avg_price
row << regionMarkupObj.mom*100
}else{
row << ""
row << ""
}
}
}else{
println d['I'] +" 未找到该id的楼盘"
}
}
row << d['C']
row << d['E']
row << d['F']
row << d['G']
rows << row
}
return rows
}
def getLastMonthSortNumer(idNum, lastMonthStr){
def q = new BasicDBObject("fjw_num", idNum)
def lastMonthObj = HONGXING_MONTH_STAT.findOne(q)
if(lastMonthObj){
def rank_map = lastMonthObj.rank_map
if(rank_map){
def detailMap = rank_map.get(lastMonthStr)
def num = null;
if(detailMap){
num = detailMap.num;
}
return num;
}
}
return null;
}
def getDistSortNumber(city, region, lastMonthStr, tihuanMap){
if(!tihuanMap){
return null
}
def rets = findDistrictPrice(city, region, lastMonthStr)
def tihuanKey = null
def tihuanValue = null
tihuanMap.each{key,value->
tihuanKey = key
tihuanValue = value
}
//对价格打85折
for(def map : rets){
def keys = null
def price = null
map.each{k,v->
keys = k
price = v
}
def newPrice = price * 0.85
if(keys == tihuanKey){
//替换价格
map.put(tihuanKey, tihuanValue)
}else{
map.put(keys, newPrice)
}
}
//排序
rets = rets.sort { a, b ->
// 获取每个map的第一个entry的value进行比较
def aValue = a.values().first()
def bValue = b.values().first()
bValue <=> aValue // 降序:b在前,a在后
}
//查找名次
def index = 0
def lastIndex = null;
for(def map : rets){
index++
def key = null
map.each{k,v->
key = k
}
if(key == tihuanKey){
lastIndex = index
break;
}
}
def resMap = [:]
resMap.put("num", lastIndex)
resMap.put("total", rets.size())
return resMap
}
def findDistrictMonthPrice(city, region, district_name, lastMonthStr){
def q = new BasicDBObject("date", lastMonthStr.toInteger()).append("city", city).append("region", region).append("district_name", district_name).append("date", lastMonthStr.toInteger())
return DISTRICT_SELL_STAT.findOne(q)
}
def findCurMonthPrice(def date){
def q = new BasicDBObject("date", date.toInteger())
return DISTRICT_SELL_STAT.count(q)
}
def findDistrictPrice(city, region, lastMonthStr){
def q = new BasicDBObject("date", lastMonthStr.toInteger()).append("city", city).append("region", region).append("date", lastMonthStr.toInteger())
def dbcursor = DISTRICT_SELL_STAT.find(q)
def rets = []
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.district_name
def map = [:]
map.put(key, d.avg_price)
rets << map
}
return rets;
}
def findRegionPrice(city, region, lastMonthStr){
def q = new BasicDBObject("date", lastMonthStr.toInteger()).append("city", city).append("region", region)
return REGION_SELL_STAT.findOne(q);
}
def findRentOfficeMarkup(city, region, lastMonthStr, cat){
def q = new BasicDBObject("date", lastMonthStr.toInteger()).append("city", city).append("cat",cat)
if(region){
q.put("district", region)
}
return OFFICE_RENT_STAT.findOne(q);
}
def findSellOfficeMarkup(city, region, lastMonthStr, cat){
def q = new BasicDBObject("date", lastMonthStr.toInteger()).append("city", city).append("cat",cat)
if(region){
q.put("district", region)
}
return OFFICE_SELL_STAT.findOne(q);
}
def findSellShopMarkup(city, region, lastMonthStr, cat){
def q = new BasicDBObject("date", lastMonthStr.toInteger()).append("city", city).append("cat", cat)
if(region){
q.put("district", region)
}
return SHOP_SELL_STAT.findOne(q);
}
}
@@ -1,15 +1,13 @@
package com.fangjia.script.jiangnanblank
package com.fangjia.script.gj.hongxing
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.PriceData
import com.fangjia.core.util.SvgUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.json.JsonBuilder
import groovy.sql.Sql
import java.util.concurrent.ExecutorService
@@ -21,9 +19,9 @@ import groovy.json.JsonOutput
class TestEvaluation {
def sql = Sql.newInstance(
"jdbc:mysql://192.168.0.92:3306/fangjiatong?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai",
"jdbc:mysql://192.168.0.94:3306/fangjiatong?useSSL=false&useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai",
"goojia",
"G0ojia123456!@#",
"G00jia123456!@#",
"com.mysql.cj.jdbc.Driver"
)
@@ -39,13 +37,19 @@ class TestEvaluation {
// task.newAndOldMatch()
// task.addFjwToBprice("D:\\数据\\宏信息\\宏信房源台账-价格.xlsx")
task.addAreaFactor()
// task.addAreaFactor()
// task.addOptName()
// task.addDistId()
// task.addEvaluationToDB()
// task.addEvaluationDetail()
task.addEvaluationDetail()
// ExcelUtils.modify("D:\\数据\\宏信息\\最新0112核价结果(红色匹配+绿色价格)宏信估值详情-.xlsx", { d ->
// if(d['Y']){
// d['Z'] = DataHandleUtils.getDoubleRound(2, d['Y']/10000.0);
// }
// })
}
def houseSimilar(){
@@ -359,11 +363,11 @@ class TestEvaluation {
def addEvaluationToDB(){
def rets = []
ExcelUtils.modify("D:\\数据\\宏信息\\宏信项目-最终.xlsx", { d ->
ExcelUtils.modify("D:\\数据\\宏信息\\宏信项目-第二版-价格修正.xlsx", { d ->
if(d['G']){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
def buildType = 3
def buildType = 0
if(d['B'] == "公寓"){
buildType = 8
}else if(d['B'] == "别墅"){
@@ -372,22 +376,24 @@ class TestEvaluation {
buildType = 1
}else if(d['B'] == "办公楼"){
buildType = 2
}else if(d['B'] == "住宅"){
buildType = 3
}
def lat = dist.lat/10000000000.0
def lng = dist.lng/10000000000.0
def cat = 6
if(buildType==3 || buildType==8 || buildType ==11){
if(buildType==3 || buildType==8 || buildType ==11 || buildType ==0){
cat = 1
}else if(buildType == 1){
cat = 4
}else if(buildType == 2){
cat = 6
}else if(buildType == 2){
cat = 4
}
def date = new Date()
rets << [d['E'], dist.city, dist.region, dist.name, dist.address,buildType,d['I'], d['J'].toString().toDouble().toInteger(), lng, lat, d['F'], date, date, cat, d['K'].toString().toDouble().toInteger(), 2]
}
}
},1)
},0)
addDataToDB(rets)
}
@@ -407,9 +413,9 @@ class TestEvaluation {
def addEvaluationDetail(){
def rets = []
def detailList = ExcelUtils.read("D:\\数据\\宏信息\\宏信房源台账-基准价-详情.xlsx", 0 , 1)
ExcelUtils.modify("D:\\数据\\宏信息\\宏信项目-最终.xlsx", { d ->
if (d['G'] && d['L'] && d['L'].toString().toDouble().toInteger()>=3472507) {
def detailList = ExcelUtils.read("D:\\数据\\宏信息\\最新0112核价结果(红色匹配+绿色价格)宏信估值详情-修改总价.xlsx", 0 , 1)
ExcelUtils.modify("D:\\数据\\宏信息\\宏信项目-第二版-价格修正.xlsx", { d ->
if (d['G'] && d['L']) {
def rows = findEvaluationIdExists(d['L'].toString().toDouble().toInteger())
if(!rows){
def dist = CommonDatautils.findDistrictById(d['G'])
@@ -417,7 +423,7 @@ class TestEvaluation {
def ret = []
ret << d['L'].toString().toDouble().toInteger()
def houses = []
if(["住宅","别墅","公寓"].contains(d["B"])){
if(["住宅","别墅","公寓","商住"].contains(d["B"])){
def propertyType = 0
if(d["B"] == "别墅"){
propertyType = 1
@@ -426,14 +432,14 @@ class TestEvaluation {
def minPrice = null
def maxPrice = null
if(avgPrice){
minPrice = avgPrice
minPrice = avgPrice.toString().toDouble().toInteger()
maxPrice = (int)(avgPrice + avgPrice * 0.1)
}
houses = OpenApiClient.newHouseAction(dist.city, dist.region, dist.name, propertyType.toString(), minPrice, maxPrice, 5)
if(houses.size()<5){
if(!houses || (houses && houses.size()<5)){
if(avgPrice){
minPrice = avgPrice
minPrice = avgPrice.toString().toDouble().toInteger()
maxPrice = (int)(avgPrice + avgPrice * 0.2)
}
houses = OpenApiClient.newHouseAction(dist.city, dist.region, dist.name, propertyType.toString(), minPrice, maxPrice, 15)
@@ -530,7 +536,8 @@ class TestEvaluation {
if(data["I"]==id && type==data['D']){
def totalPrice = 0
if (data['H'] && data['N']){
totalPrice = (int)(data['H'] * data['N'])
totalPrice = data['H'] * data['N']
totalPrice = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, totalPrice/10000.0)
}
def unitPrice = 0
if(data['N']){
@@ -10,10 +10,10 @@ class ShBankVillage {
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\上海银行\\区域判定-to估家.xlsx", { d ->
ExcelUtils.multiThreadingModify("D:\\数据\\上海银行\\涉农-2月估家数据反馈.xlsx", { d ->
if (!d['R'] && d['B']) {
def dto = OpenApiUtils.addrVerifyShBank(null, d['B'])
if (!d['R'] && d['C']) {
def dto = OpenApiUtils.addrVerifyShBank(d['B'], d['C'])
if (dto) {
// def dto = jsonObj.result
d['E'] = dto.remark
@@ -11,7 +11,7 @@ import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
//67423021cae3b58e5c2ac595a389b036 :临时买的key
def static GAO_DE_AK = "d22fcd4274e037fdd1e9357d91152504" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
def static GAO_DE_AK = "b3aa946af7979e9e9dfca83f1b7036b0" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/**
@@ -34,12 +34,25 @@ class BasicInfoDataSynch extends BaseBatch{
@Override
public void process(Object args) {
synchCitys();
synchRegion()
synchTown()
synchDict()
// synchCitys();
// synchRegion()
// synchTown()
// synchDict()
// synchDistrict("district")
// synchDistrict("office")
def q = new BasicDBObject("city","重庆").append("region","两江")
def dbcursor = ZS_SEAWEED_CONN.find(q)
println dbcursor.count()
for(def d : dbcursor){
def dbo = CommonDatautils.findDistrictById(d.get("_id").toString())
if(dbo){
def udata = new BasicDBObject()
udata.put("region",dbo.region)
ZS_SEAWEED_CONN.update(new BasicDBObject("_id",d.get("_id")), new BasicDBObject('$set',udata))
}
}
}
def synchCitys(){