供数200个福州小区脚本

This commit is contained in:
2025-09-15 13:09:24 +08:00
parent 708fde620d
commit 63cc2e734e
3 changed files with 508 additions and 11 deletions
@@ -4905,6 +4905,10 @@ class CommonDatautils {
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if (!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price > 3000) {
dataMap.put("soufang_esf_price", db.soufang_esf_price)
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
@@ -1,19 +1,12 @@
package com.fangjia.script.fuzhou
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import com.mongodb.*
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
/**
* 福州数据结果统计
*/
@@ -89,7 +82,7 @@ class FzDataResultStat {
map2.put(key, obj)
}
def rows = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
rows << ["ID","城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
def index = 0
map1.each {k,obj1->
def keys = k.split("##")
@@ -114,7 +107,7 @@ class FzDataResultStat {
if(markup){
markup_abs = Math.abs(markup)
}
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
def row = [obj1._id.toString(), obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
rows << row
index++
println row
@@ -0,0 +1,500 @@
package com.fangjia.script.majintao.FuZhou
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.atomic.AtomicInteger
class GongShu {
static main(args) {
def task = new GongShu();
task.run();
}
def run() {
String path1 = "D:\\数据\\福州\\福州200个小区抽样价格核验202509.xlsx";
// String path1 = "D:\\数据\\福州\\价格比对_20240915.xlsx";
String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
// String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
String path3 = "E:\\goojia\\file\\excel文件\\福州\\福州贝壳小区.xlsx";
// String path2 = "E:\\goojia\\file\\excel文件\\福州\\1.xlsx";
// String path1 = "E:\\goojia\\file\\excel文件\\福州\\福州全部小区.xlsx";
// findID(path2)
// findDelDistrict(path2)
// findDistrictByAlias(path2)
// match19UseMAP(path2)
// addFjwPrice(path2)
// match(path2, path1)
// match2(path3, path1)
// houseCase(path2)
thirdPrice(path1)
// priceAvg(path1)
// futurePrice(path2)
// rePrice(path2)
}
def findID(path2) {
def count = new AtomicInteger(0)
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def city = d['C']
def region = d['D']
def name = d['E']
def district = CommonDatautils.findDistrict(city, region, name, "district")
if (district) {
d['B'] = district._id.toString()
}
})
}
def findDelDistrict(path2) {
def count = new AtomicInteger(0)
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def city = d['C']
def region = d['D']
def name = d['E']
def district = CommonDatautils.findDistrictAll(city, region, name, "district")
if ("1" == district.status.toString()) {
// d['B'] = district._id.toString()
d['G'] = "已删除"
}
})
}
def findDistrictByAlias(path2) {
def count = new AtomicInteger(0)
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
if (d['G']) {
def city = d['C']
def region = d['D']
def name = d['E']
def district = CommonDatautils.findSeaweedByAlias(city, region, name, "district")
if (district) {
d['H'] = district.get(0)._id.toString()
}
}
})
}
def match19UseMAP(path1) {
def count = new AtomicInteger(0)
ExcelUtils.defineByInputThreadingModify(10, path1, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def address = d['B'] + d['D'] + d['C']
def city = d['B']
def name = d['C']
def region = d['D']
//19匹配
String url = "http://192.168.0.19:31080/FangJiaNlpService/match";
def map = [:]
map.put("address", address)
map.put("city", city)
map.put("useMap", "true")
map.put("category", "property")
HttpRequest httpRequest = new HttpRequest();
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content
if (result != null && "name" in result) {
if ("city" in result) {
d['G'] = result.get("city").toString().replaceAll("null", "")
}
if ("district" in result) {
d['H'] = result.get("district").toString().replaceAll("null", "")
}
if ("name" in result) {
d['I'] = result.get("name").toString().replaceAll("null", "")
}
} else {
address = d['B'] + d['D'] + d['A']
map.put("address", address)
def content2 = httpRequest.getJsonContent(url, map, "UTF-8")
def result2 = content2
if (result2 != null && "name" in result2) {
if ("city" in result2) {
d['G'] = result2.get("city").toString().replaceAll("null", "")
}
if ("district" in result2) {
d['H'] = result2.get("district").toString().replaceAll("null", "")
}
if ("name" in result2) {
d['I'] = result2.get("name").toString().replaceAll("null", "")
}
}
}
})
}
def addFjwPrice(path1) {
ExcelUtils.defineByInputThreadingModify(5, path1, { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if (dist) {
def price = CommonDatautils.queryThisMonthDistrictAvgPrice(dist, "2024-11-01")
if (price) {
d['F'] = price.avg_price
d['G'] = price.avg_price_2b
def dbo = price
// 计算方式
if (dbo.value_type == 1) {
dbo.cal_type = "指导价"
} else if (dbo.value_type == 2) {
dbo.cal_type = "自身样本"
} else if (dbo.value_type == 3) {
dbo.cal_type = "第三方"
} else if (dbo.value_type == 4) {
dbo.cal_type = "交易数据"
} else if (dbo.value_type == 5) {
dbo.cal_type = "周边楼盘"
} else if (dbo.value_type == 6) {
dbo.cal_type = "推导"
} else if (dbo.value_type == 9) {
dbo.cal_type = "神经网络"
} else if (dbo.value_type == 10) {
dbo.cal_type = "无方式"
} else {
dbo.cal_type = ""
}
d['I'] = dbo.cal_type
}
}
}
})
}
def match2(path2, path1) {
def count = new AtomicInteger(0)
def map = [:]
ExcelUtils.removeEmptyRow(path1)
ExcelUtils.removeEmptyRow(path2)
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def key = d['G'] + d['H'] + d['I']
map.put(key, d)
})
print(map)
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
def key = "福州" + d['C'] + d['D']
if (map.containsKey(key)) {
def value = map.get(key)
d['J'] = value['B']
d['K'] = value['D']
d['L'] = value['C']
}
})
}
def match(path2, path1) {
def count = new AtomicInteger(0)
def map = [:]
ExcelUtils.removeEmptyRow(path1)
ExcelUtils.removeEmptyRow(path2)
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def key = d['C'] + d['D'] + d['E']
map.put(key, d)
})
print(map)
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
def key = "福州" + d['C'] + d['D']
if (map.containsKey(key)) {
def value = map.get(key)
if (!d['J']) {
d['J'] = value['J']
d['K'] = value['K']
d['L'] = value['L']
if (value['J']) {
d['M'] = "pp"
}
}
}
})
}
// 房源及数量
def houseCase(path2) {
def count = new AtomicInteger()
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def price = []
def city
def region
def name
if (d['J'] && d['K'] && d['L']) {
city = d['J']
region = d['K']
name = d['L']
def must = []
def term1 = ["terms": ["root": ["ke.com"]]] // 修正terms查询的格式
must << term1
def term2 = ["terms": ["cat": ["sell"]]]
must << term2
def term3 = ["terms": ["city": [city]]]
must << term3
def term4 = ["terms": ["region": [region]]]
must << term4
def term5 = ["terms": ["district_name": [name]]]
must << term5
def params = ["query": ["bool": ["must": must]]]
// 添加 p_date 大于 20240101 的条件
def term6 = ["range": ["p_date": ["gt": "20240401"]]]
must << term6
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/seawater-20*/_search");
def total = 0
if (s) {
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if (hits) {
total = hits.total.value
if (hits.hits.size() > 0) {
hits.hits.each { hit ->
if (hit._source && hit._source.avg_price) {
price.add(hit._source.avg_price)
}
}
}
}
}
if (price.size() > 0) {
println(price)
// 计算中位数
def medianPrice = calculateMedian(price)
// 找到与中位数最接近的有效价格
def nearestPrice = findNearestPrice(medianPrice, price)
print(nearestPrice)
d['O'] = nearestPrice
d['P'] = total
}
}
})
}
//查找三方价格
def thirdPrice(path1) {
def count = 0
ExcelUtils.multiThreadingModify( path1, { List d ->
count++
if (count % 10 == 0) {
println("已处理:" + count)
}
def city = "福州"
def region = d['B']
def name = d['C']
def valusMap = CommonDatautils.findThirdPartyPrice(city, region, name, 202505, 202508)
if (valusMap) {
def dataMap = valusMap.get(city + "##" + region + "##" + name)
if (dataMap.get("anjuke_esf_price")) {
d['F'] = (int)(dataMap.get("anjuke_esf_price") * 0.9)
}
if (dataMap.get("maitian_esf_price")) {
d['G'] = (int)(dataMap.get("maitian_esf_price") * 0.9)
}
if (dataMap.get("soufang_esf_price")) {
d['H'] = (int)(dataMap.get("soufang_esf_price") * 0.9)
}
if (dataMap.get("cityhouse_esf_price")) {
d['I'] = (int)(dataMap.get("cityhouse_esf_price") * 0.9)
}
}
})
}
def priceAvg(path1) {
def count = 0
ExcelUtils.multiThreadingModify(path1, { List d ->
count++
if (count % 10 == 0) {
println("已处理:" + count)
}
if (d['B']) {
def city
def region
def name
if (d['B'] && d['C']) {
city ="福州"
region = d['B']
name = d['C']
String sql = "select * from community where city = '${city}' and region = '${region}' and name.keyword = '${name}' " +
"and source = '贝壳找房' ";
ElasticUtils.executeSqlQuery(sql, { result ->
if (result.size() > 0) {
println(name +" "+result.avg_price)
if ("avg_price" in result) {
d['N'] = result.avg_price
}
if ("half_year_deal_avg_price" in result) {
d['E'] = result.half_year_deal_avg_price
}
}
})
}
}
})
}
def futurePrice(path1) {
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
def count = 0
ExcelUtils.removeEmptyRow(path1)
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
count++
if (count % 10 == 0) {
println("已处理:" + count)
}
if (d['C'] && d['D'] && d['E']) {
String city = d['C']
String region = d['D']
String name = d['E']
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject("weekend", -1)).limit(1)
def futurePrice
for (def db : dbcursor) {
if (db.future_value) {
futurePrice = db.future_value
d['H'] = futurePrice
break;
}
}
}
})
}
//重算小区价格
def rePrice(path2) {
ExcelUtils.removeEmptyRow(path2)
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
if (d['A'] && d['B'] && d['C']) {
String city = d['A']
String region = d['B']
String name = d['C']
// CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240721", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240915", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240922", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240929", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20241006", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20241013", "yyyyMMdd"))
}
})
}
// 计算中位数
def calculateMedian(prices) {
def sortedPrices = prices.sort()
def size = sortedPrices.size()
if (size % 2 == 0) {
// 如果列表长度为偶数,则取中间两个数的平均值作为中位数
def midIndex = (size / 2).toInteger()
return (sortedPrices[midIndex - 1] + sortedPrices[midIndex]) / 2.0
} else {
// 如果列表长度为奇数,则取中间的数作为中位数
return sortedPrices[((size - 1) / 2).toInteger()]
}
}
def findNearestPrice(targetPrice, validPrices) {
def nearestPrice = validPrices.first() // 假设第一个有效价格是最接近的价格
def minDiff = Math.abs(targetPrice - nearestPrice)
validPrices.each { price ->
def diff = Math.abs(targetPrice - price)
if (diff < minDiff) {
minDiff = diff
nearestPrice = price
}
}
return nearestPrice
}
def match235(path2) {
DBCollection DATA_PROCESS = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018);
def count = 0
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
count++
if (count % 10 == 0) {
println("已处理:" + count)
}
def matchCity = d['B']
def matchRegion = d['C']
def matchName = d['D']
def q = new BasicDBObject();
q.put("match_city", matchCity)
q.put("match_region", matchRegion)
q.put("match_name", matchName)
q.put("source", "链家")
q.put("s_date", 202404)
def data = DATA_PROCESS.find(q)
for (da in data) {
if (da) {
def city = da.city
def region = da.region
def name = da.name
d['G'] = city
d['H'] = region
d['I'] = name
}
}
})
}
}