供数200个福州小区脚本
This commit is contained in:
+4
@@ -4905,6 +4905,10 @@ class CommonDatautils {
|
||||
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
|
||||
}
|
||||
|
||||
if (!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price > 3000) {
|
||||
dataMap.put("soufang_esf_price", db.soufang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
|
||||
}
|
||||
|
||||
+4
-11
@@ -1,19 +1,12 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.AMQP
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 福州数据结果统计
|
||||
*/
|
||||
@@ -89,7 +82,7 @@ class FzDataResultStat {
|
||||
map2.put(key, obj)
|
||||
}
|
||||
def rows = []
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
|
||||
rows << ["ID","城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
|
||||
def index = 0
|
||||
map1.each {k,obj1->
|
||||
def keys = k.split("##")
|
||||
@@ -114,7 +107,7 @@ class FzDataResultStat {
|
||||
if(markup){
|
||||
markup_abs = Math.abs(markup)
|
||||
}
|
||||
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
|
||||
def row = [obj1._id.toString(), obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
|
||||
rows << row
|
||||
index++
|
||||
println row
|
||||
|
||||
@@ -0,0 +1,500 @@
|
||||
package com.fangjia.script.majintao.FuZhou
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class GongShu {
|
||||
static main(args) {
|
||||
def task = new GongShu();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run() {
|
||||
String path1 = "D:\\数据\\福州\\福州200个小区抽样价格核验202509.xlsx";
|
||||
// String path1 = "D:\\数据\\福州\\价格比对_20240915.xlsx";
|
||||
String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
|
||||
// String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
|
||||
String path3 = "E:\\goojia\\file\\excel文件\\福州\\福州贝壳小区.xlsx";
|
||||
// String path2 = "E:\\goojia\\file\\excel文件\\福州\\1.xlsx";
|
||||
// String path1 = "E:\\goojia\\file\\excel文件\\福州\\福州全部小区.xlsx";
|
||||
// findID(path2)
|
||||
// findDelDistrict(path2)
|
||||
// findDistrictByAlias(path2)
|
||||
// match19UseMAP(path2)
|
||||
// addFjwPrice(path2)
|
||||
|
||||
// match(path2, path1)
|
||||
// match2(path3, path1)
|
||||
// houseCase(path2)
|
||||
thirdPrice(path1)
|
||||
// priceAvg(path1)
|
||||
// futurePrice(path2)
|
||||
// rePrice(path2)
|
||||
}
|
||||
|
||||
def findID(path2) {
|
||||
def count = new AtomicInteger(0)
|
||||
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
|
||||
count.incrementAndGet()
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def city = d['C']
|
||||
def region = d['D']
|
||||
def name = d['E']
|
||||
def district = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
if (district) {
|
||||
d['B'] = district._id.toString()
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def findDelDistrict(path2) {
|
||||
def count = new AtomicInteger(0)
|
||||
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
|
||||
count.incrementAndGet()
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def city = d['C']
|
||||
def region = d['D']
|
||||
def name = d['E']
|
||||
def district = CommonDatautils.findDistrictAll(city, region, name, "district")
|
||||
if ("1" == district.status.toString()) {
|
||||
// d['B'] = district._id.toString()
|
||||
d['G'] = "已删除"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findDistrictByAlias(path2) {
|
||||
def count = new AtomicInteger(0)
|
||||
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
|
||||
count.incrementAndGet()
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
if (d['G']) {
|
||||
def city = d['C']
|
||||
def region = d['D']
|
||||
def name = d['E']
|
||||
def district = CommonDatautils.findSeaweedByAlias(city, region, name, "district")
|
||||
if (district) {
|
||||
d['H'] = district.get(0)._id.toString()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match19UseMAP(path1) {
|
||||
def count = new AtomicInteger(0)
|
||||
ExcelUtils.defineByInputThreadingModify(10, path1, { List d ->
|
||||
count.incrementAndGet()
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def address = d['B'] + d['D'] + d['C']
|
||||
def city = d['B']
|
||||
def name = d['C']
|
||||
def region = d['D']
|
||||
|
||||
//19匹配
|
||||
String url = "http://192.168.0.19:31080/FangJiaNlpService/match";
|
||||
def map = [:]
|
||||
map.put("address", address)
|
||||
map.put("city", city)
|
||||
map.put("useMap", "true")
|
||||
map.put("category", "property")
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
def content = httpRequest.getJsonContent(url, map, "UTF-8")
|
||||
def result = content
|
||||
if (result != null && "name" in result) {
|
||||
if ("city" in result) {
|
||||
d['G'] = result.get("city").toString().replaceAll("null", "")
|
||||
}
|
||||
if ("district" in result) {
|
||||
d['H'] = result.get("district").toString().replaceAll("null", "")
|
||||
}
|
||||
if ("name" in result) {
|
||||
d['I'] = result.get("name").toString().replaceAll("null", "")
|
||||
}
|
||||
|
||||
} else {
|
||||
address = d['B'] + d['D'] + d['A']
|
||||
map.put("address", address)
|
||||
def content2 = httpRequest.getJsonContent(url, map, "UTF-8")
|
||||
def result2 = content2
|
||||
if (result2 != null && "name" in result2) {
|
||||
if ("city" in result2) {
|
||||
d['G'] = result2.get("city").toString().replaceAll("null", "")
|
||||
}
|
||||
if ("district" in result2) {
|
||||
d['H'] = result2.get("district").toString().replaceAll("null", "")
|
||||
}
|
||||
if ("name" in result2) {
|
||||
d['I'] = result2.get("name").toString().replaceAll("null", "")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def addFjwPrice(path1) {
|
||||
ExcelUtils.defineByInputThreadingModify(5, path1, { d ->
|
||||
if (d['B']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if (dist) {
|
||||
def price = CommonDatautils.queryThisMonthDistrictAvgPrice(dist, "2024-11-01")
|
||||
if (price) {
|
||||
d['F'] = price.avg_price
|
||||
d['G'] = price.avg_price_2b
|
||||
def dbo = price
|
||||
// 计算方式
|
||||
if (dbo.value_type == 1) {
|
||||
dbo.cal_type = "指导价"
|
||||
} else if (dbo.value_type == 2) {
|
||||
dbo.cal_type = "自身样本"
|
||||
} else if (dbo.value_type == 3) {
|
||||
dbo.cal_type = "第三方"
|
||||
} else if (dbo.value_type == 4) {
|
||||
dbo.cal_type = "交易数据"
|
||||
} else if (dbo.value_type == 5) {
|
||||
dbo.cal_type = "周边楼盘"
|
||||
} else if (dbo.value_type == 6) {
|
||||
dbo.cal_type = "推导"
|
||||
} else if (dbo.value_type == 9) {
|
||||
dbo.cal_type = "神经网络"
|
||||
} else if (dbo.value_type == 10) {
|
||||
dbo.cal_type = "无方式"
|
||||
} else {
|
||||
dbo.cal_type = ""
|
||||
}
|
||||
d['I'] = dbo.cal_type
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def match2(path2, path1) {
|
||||
def count = new AtomicInteger(0)
|
||||
def map = [:]
|
||||
ExcelUtils.removeEmptyRow(path1)
|
||||
ExcelUtils.removeEmptyRow(path2)
|
||||
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
|
||||
count.incrementAndGet()
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def key = d['G'] + d['H'] + d['I']
|
||||
map.put(key, d)
|
||||
})
|
||||
print(map)
|
||||
|
||||
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
|
||||
def key = "福州" + d['C'] + d['D']
|
||||
if (map.containsKey(key)) {
|
||||
def value = map.get(key)
|
||||
d['J'] = value['B']
|
||||
d['K'] = value['D']
|
||||
d['L'] = value['C']
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match(path2, path1) {
|
||||
def count = new AtomicInteger(0)
|
||||
def map = [:]
|
||||
ExcelUtils.removeEmptyRow(path1)
|
||||
ExcelUtils.removeEmptyRow(path2)
|
||||
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
|
||||
count.incrementAndGet()
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def key = d['C'] + d['D'] + d['E']
|
||||
map.put(key, d)
|
||||
})
|
||||
print(map)
|
||||
|
||||
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
|
||||
def key = "福州" + d['C'] + d['D']
|
||||
if (map.containsKey(key)) {
|
||||
def value = map.get(key)
|
||||
if (!d['J']) {
|
||||
d['J'] = value['J']
|
||||
d['K'] = value['K']
|
||||
d['L'] = value['L']
|
||||
if (value['J']) {
|
||||
d['M'] = "pp"
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// 房源及数量
|
||||
def houseCase(path2) {
|
||||
def count = new AtomicInteger()
|
||||
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
|
||||
count.incrementAndGet()
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def price = []
|
||||
|
||||
def city
|
||||
def region
|
||||
def name
|
||||
if (d['J'] && d['K'] && d['L']) {
|
||||
city = d['J']
|
||||
region = d['K']
|
||||
name = d['L']
|
||||
|
||||
def must = []
|
||||
def term1 = ["terms": ["root": ["ke.com"]]] // 修正terms查询的格式
|
||||
must << term1
|
||||
def term2 = ["terms": ["cat": ["sell"]]]
|
||||
must << term2
|
||||
def term3 = ["terms": ["city": [city]]]
|
||||
must << term3
|
||||
def term4 = ["terms": ["region": [region]]]
|
||||
must << term4
|
||||
def term5 = ["terms": ["district_name": [name]]]
|
||||
must << term5
|
||||
def params = ["query": ["bool": ["must": must]]]
|
||||
|
||||
// 添加 p_date 大于 20240101 的条件
|
||||
def term6 = ["range": ["p_date": ["gt": "20240401"]]]
|
||||
must << term6
|
||||
|
||||
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/seawater-20*/_search");
|
||||
def total = 0
|
||||
if (s) {
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if (hits) {
|
||||
total = hits.total.value
|
||||
if (hits.hits.size() > 0) {
|
||||
hits.hits.each { hit ->
|
||||
if (hit._source && hit._source.avg_price) {
|
||||
price.add(hit._source.avg_price)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
if (price.size() > 0) {
|
||||
println(price)
|
||||
|
||||
// 计算中位数
|
||||
def medianPrice = calculateMedian(price)
|
||||
// 找到与中位数最接近的有效价格
|
||||
def nearestPrice = findNearestPrice(medianPrice, price)
|
||||
print(nearestPrice)
|
||||
d['O'] = nearestPrice
|
||||
d['P'] = total
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
//查找三方价格
|
||||
def thirdPrice(path1) {
|
||||
def count = 0
|
||||
ExcelUtils.multiThreadingModify( path1, { List d ->
|
||||
count++
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def city = "福州"
|
||||
def region = d['B']
|
||||
def name = d['C']
|
||||
def valusMap = CommonDatautils.findThirdPartyPrice(city, region, name, 202505, 202508)
|
||||
if (valusMap) {
|
||||
def dataMap = valusMap.get(city + "##" + region + "##" + name)
|
||||
if (dataMap.get("anjuke_esf_price")) {
|
||||
d['F'] = (int)(dataMap.get("anjuke_esf_price") * 0.9)
|
||||
}
|
||||
if (dataMap.get("maitian_esf_price")) {
|
||||
d['G'] = (int)(dataMap.get("maitian_esf_price") * 0.9)
|
||||
}
|
||||
if (dataMap.get("soufang_esf_price")) {
|
||||
d['H'] = (int)(dataMap.get("soufang_esf_price") * 0.9)
|
||||
}
|
||||
if (dataMap.get("cityhouse_esf_price")) {
|
||||
d['I'] = (int)(dataMap.get("cityhouse_esf_price") * 0.9)
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def priceAvg(path1) {
|
||||
def count = 0
|
||||
ExcelUtils.multiThreadingModify(path1, { List d ->
|
||||
count++
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
if (d['B']) {
|
||||
def city
|
||||
def region
|
||||
def name
|
||||
if (d['B'] && d['C']) {
|
||||
city ="福州"
|
||||
region = d['B']
|
||||
name = d['C']
|
||||
String sql = "select * from community where city = '${city}' and region = '${region}' and name.keyword = '${name}' " +
|
||||
"and source = '贝壳找房' ";
|
||||
ElasticUtils.executeSqlQuery(sql, { result ->
|
||||
if (result.size() > 0) {
|
||||
println(name +" "+result.avg_price)
|
||||
if ("avg_price" in result) {
|
||||
d['N'] = result.avg_price
|
||||
}
|
||||
if ("half_year_deal_avg_price" in result) {
|
||||
d['E'] = result.half_year_deal_avg_price
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def futurePrice(path1) {
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
def count = 0
|
||||
ExcelUtils.removeEmptyRow(path1)
|
||||
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
|
||||
count++
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
if (d['C'] && d['D'] && d['E']) {
|
||||
String city = d['C']
|
||||
String region = d['D']
|
||||
String name = d['E']
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
def futurePrice
|
||||
for (def db : dbcursor) {
|
||||
if (db.future_value) {
|
||||
futurePrice = db.future_value
|
||||
d['H'] = futurePrice
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
//重算小区价格
|
||||
def rePrice(path2) {
|
||||
ExcelUtils.removeEmptyRow(path2)
|
||||
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
|
||||
if (d['A'] && d['B'] && d['C']) {
|
||||
String city = d['A']
|
||||
String region = d['B']
|
||||
String name = d['C']
|
||||
// CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240721", "yyyyMMdd"))
|
||||
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240915", "yyyyMMdd"))
|
||||
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240922", "yyyyMMdd"))
|
||||
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240929", "yyyyMMdd"))
|
||||
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20241006", "yyyyMMdd"))
|
||||
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20241013", "yyyyMMdd"))
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// 计算中位数
|
||||
def calculateMedian(prices) {
|
||||
def sortedPrices = prices.sort()
|
||||
def size = sortedPrices.size()
|
||||
if (size % 2 == 0) {
|
||||
// 如果列表长度为偶数,则取中间两个数的平均值作为中位数
|
||||
def midIndex = (size / 2).toInteger()
|
||||
return (sortedPrices[midIndex - 1] + sortedPrices[midIndex]) / 2.0
|
||||
} else {
|
||||
// 如果列表长度为奇数,则取中间的数作为中位数
|
||||
return sortedPrices[((size - 1) / 2).toInteger()]
|
||||
}
|
||||
}
|
||||
|
||||
def findNearestPrice(targetPrice, validPrices) {
|
||||
def nearestPrice = validPrices.first() // 假设第一个有效价格是最接近的价格
|
||||
def minDiff = Math.abs(targetPrice - nearestPrice)
|
||||
|
||||
validPrices.each { price ->
|
||||
def diff = Math.abs(targetPrice - price)
|
||||
if (diff < minDiff) {
|
||||
minDiff = diff
|
||||
nearestPrice = price
|
||||
}
|
||||
}
|
||||
|
||||
return nearestPrice
|
||||
}
|
||||
|
||||
def match235(path2) {
|
||||
DBCollection DATA_PROCESS = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018);
|
||||
|
||||
def count = 0
|
||||
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
|
||||
count++
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
def matchCity = d['B']
|
||||
def matchRegion = d['C']
|
||||
def matchName = d['D']
|
||||
def q = new BasicDBObject();
|
||||
q.put("match_city", matchCity)
|
||||
q.put("match_region", matchRegion)
|
||||
q.put("match_name", matchName)
|
||||
q.put("source", "链家")
|
||||
q.put("s_date", 202404)
|
||||
def data = DATA_PROCESS.find(q)
|
||||
for (da in data) {
|
||||
if (da) {
|
||||
def city = da.city
|
||||
def region = da.region
|
||||
def name = da.name
|
||||
d['G'] = city
|
||||
d['H'] = region
|
||||
d['I'] = name
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user