diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 955fc06e2..c64ee5c4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4930,6 +4930,10 @@ class CommonDatautils { if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) } + + if (!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price > 3000) { + dataMap.put("soufang_esf_price", db.soufang_esf_price) + } if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 796876790..ef11f554b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -1,19 +1,12 @@ package com.fangjia.script.fuzhou -import com.fangjia.script.commData.CommonDatautils + import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DBCollection -import com.mongodb.DBCursor -import com.mongodb.DBObject -import com.rabbitmq.client.AMQP -import com.rabbitmq.client.impl.AMQImpl +import com.mongodb.* import org.apache.commons.io.FileUtils import org.bson.types.ObjectId - /** * 福州数据结果统计 */ @@ -103,7 +96,7 @@ class FzDataResultStat { map2.put(key, obj) } def rows = [] - rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"] + rows << ["ID","城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"] def index = 0 map1.each {k,obj1-> def keys = k.split("##") @@ -128,7 +121,7 @@ class FzDataResultStat { if(markup){ markup_abs = Math.abs(markup) } - def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs] + def row = [obj1._id.toString(), obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs] rows << row index++ println row diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/GongShu.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/GongShu.groovy new file mode 100644 index 000000000..9e73b0422 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/GongShu.groovy @@ -0,0 +1,500 @@ +package com.fangjia.script.majintao.FuZhou + +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.atomic.AtomicInteger + +class GongShu { + static main(args) { + def task = new GongShu(); + task.run(); + } + + def run() { + String path1 = "D:\\数据\\福州\\福州200个小区抽样价格核验202509.xlsx"; +// String path1 = "D:\\数据\\福州\\价格比对_20240915.xlsx"; + String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx"; +// String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx"; + String path3 = "E:\\goojia\\file\\excel文件\\福州\\福州贝壳小区.xlsx"; +// String path2 = "E:\\goojia\\file\\excel文件\\福州\\1.xlsx"; +// String path1 = "E:\\goojia\\file\\excel文件\\福州\\福州全部小区.xlsx"; +// findID(path2) +// findDelDistrict(path2) +// findDistrictByAlias(path2) +// match19UseMAP(path2) +// addFjwPrice(path2) + +// match(path2, path1) +// match2(path3, path1) +// houseCase(path2) + thirdPrice(path1) +// priceAvg(path1) +// futurePrice(path2) +// rePrice(path2) + } + + def findID(path2) { + def count = new AtomicInteger(0) + ExcelUtils.defineByInputThreadingModify(10, path2, { List d -> + count.incrementAndGet() + if (count % 10 == 0) { + println("已处理:" + count) + } + def city = d['C'] + def region = d['D'] + def name = d['E'] + def district = CommonDatautils.findDistrict(city, region, name, "district") + if (district) { + d['B'] = district._id.toString() + } + }) + + } + + def findDelDistrict(path2) { + def count = new AtomicInteger(0) + ExcelUtils.defineByInputThreadingModify(10, path2, { List d -> + count.incrementAndGet() + if (count % 10 == 0) { + println("已处理:" + count) + } + def city = d['C'] + def region = d['D'] + def name = d['E'] + def district = CommonDatautils.findDistrictAll(city, region, name, "district") + if ("1" == district.status.toString()) { +// d['B'] = district._id.toString() + d['G'] = "已删除" + } + }) + } + + def findDistrictByAlias(path2) { + def count = new AtomicInteger(0) + ExcelUtils.defineByInputThreadingModify(10, path2, { List d -> + count.incrementAndGet() + if (count % 10 == 0) { + println("已处理:" + count) + } + if (d['G']) { + def city = d['C'] + def region = d['D'] + def name = d['E'] + def district = CommonDatautils.findSeaweedByAlias(city, region, name, "district") + if (district) { + d['H'] = district.get(0)._id.toString() + } + } + }) + } + + def match19UseMAP(path1) { + def count = new AtomicInteger(0) + ExcelUtils.defineByInputThreadingModify(10, path1, { List d -> + count.incrementAndGet() + if (count % 10 == 0) { + println("已处理:" + count) + } + def address = d['B'] + d['D'] + d['C'] + def city = d['B'] + def name = d['C'] + def region = d['D'] + + //19匹配 + String url = "http://192.168.0.19:31080/FangJiaNlpService/match"; + def map = [:] + map.put("address", address) + map.put("city", city) + map.put("useMap", "true") + map.put("category", "property") + HttpRequest httpRequest = new HttpRequest(); + def content = httpRequest.getJsonContent(url, map, "UTF-8") + def result = content + if (result != null && "name" in result) { + if ("city" in result) { + d['G'] = result.get("city").toString().replaceAll("null", "") + } + if ("district" in result) { + d['H'] = result.get("district").toString().replaceAll("null", "") + } + if ("name" in result) { + d['I'] = result.get("name").toString().replaceAll("null", "") + } + + } else { + address = d['B'] + d['D'] + d['A'] + map.put("address", address) + def content2 = httpRequest.getJsonContent(url, map, "UTF-8") + def result2 = content2 + if (result2 != null && "name" in result2) { + if ("city" in result2) { + d['G'] = result2.get("city").toString().replaceAll("null", "") + } + if ("district" in result2) { + d['H'] = result2.get("district").toString().replaceAll("null", "") + } + if ("name" in result2) { + d['I'] = result2.get("name").toString().replaceAll("null", "") + } + + } + } + }) + + } + + def addFjwPrice(path1) { + ExcelUtils.defineByInputThreadingModify(5, path1, { d -> + if (d['B']) { + def dist = CommonDatautils.findDistrictById(d['B']) + if (dist) { + def price = CommonDatautils.queryThisMonthDistrictAvgPrice(dist, "2024-11-01") + if (price) { + d['F'] = price.avg_price + d['G'] = price.avg_price_2b + def dbo = price + // 计算方式 + if (dbo.value_type == 1) { + dbo.cal_type = "指导价" + } else if (dbo.value_type == 2) { + dbo.cal_type = "自身样本" + } else if (dbo.value_type == 3) { + dbo.cal_type = "第三方" + } else if (dbo.value_type == 4) { + dbo.cal_type = "交易数据" + } else if (dbo.value_type == 5) { + dbo.cal_type = "周边楼盘" + } else if (dbo.value_type == 6) { + dbo.cal_type = "推导" + } else if (dbo.value_type == 9) { + dbo.cal_type = "神经网络" + } else if (dbo.value_type == 10) { + dbo.cal_type = "无方式" + } else { + dbo.cal_type = "" + } + d['I'] = dbo.cal_type + } + + } + } + + }) + } + + def match2(path2, path1) { + def count = new AtomicInteger(0) + def map = [:] + ExcelUtils.removeEmptyRow(path1) + ExcelUtils.removeEmptyRow(path2) + ExcelUtils.defineByInputThreadingModify(1, path2, { List d -> + count.incrementAndGet() + if (count % 10 == 0) { + println("已处理:" + count) + } + def key = d['G'] + d['H'] + d['I'] + map.put(key, d) + }) + print(map) + + ExcelUtils.defineByInputThreadingModify(1, path1, { List d -> + def key = "福州" + d['C'] + d['D'] + if (map.containsKey(key)) { + def value = map.get(key) + d['J'] = value['B'] + d['K'] = value['D'] + d['L'] = value['C'] + + } + }) + } + + def match(path2, path1) { + def count = new AtomicInteger(0) + def map = [:] + ExcelUtils.removeEmptyRow(path1) + ExcelUtils.removeEmptyRow(path2) + ExcelUtils.defineByInputThreadingModify(1, path2, { List d -> + count.incrementAndGet() + if (count % 10 == 0) { + println("已处理:" + count) + } + def key = d['C'] + d['D'] + d['E'] + map.put(key, d) + }) + print(map) + + ExcelUtils.defineByInputThreadingModify(1, path1, { List d -> + def key = "福州" + d['C'] + d['D'] + if (map.containsKey(key)) { + def value = map.get(key) + if (!d['J']) { + d['J'] = value['J'] + d['K'] = value['K'] + d['L'] = value['L'] + if (value['J']) { + d['M'] = "pp" + } + } + } + }) + + } + + // 房源及数量 + def houseCase(path2) { + def count = new AtomicInteger() + ExcelUtils.defineByInputThreadingModify(10, path2, { List d -> + count.incrementAndGet() + if (count % 10 == 0) { + println("已处理:" + count) + } + def price = [] + + def city + def region + def name + if (d['J'] && d['K'] && d['L']) { + city = d['J'] + region = d['K'] + name = d['L'] + + def must = [] + def term1 = ["terms": ["root": ["ke.com"]]] // 修正terms查询的格式 + must << term1 + def term2 = ["terms": ["cat": ["sell"]]] + must << term2 + def term3 = ["terms": ["city": [city]]] + must << term3 + def term4 = ["terms": ["region": [region]]] + must << term4 + def term5 = ["terms": ["district_name": [name]]] + must << term5 + def params = ["query": ["bool": ["must": must]]] + + // 添加 p_date 大于 20240101 的条件 + def term6 = ["range": ["p_date": ["gt": "20240401"]]] + must << term6 + + def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/seawater-20*/_search"); + def total = 0 + if (s) { + def doc = new groovy.json.JsonSlurper().parseText(s) + def hits = doc.hits + if (hits) { + total = hits.total.value + if (hits.hits.size() > 0) { + hits.hits.each { hit -> + if (hit._source && hit._source.avg_price) { + price.add(hit._source.avg_price) + } + } + } + + } + } + if (price.size() > 0) { + println(price) + + // 计算中位数 + def medianPrice = calculateMedian(price) + // 找到与中位数最接近的有效价格 + def nearestPrice = findNearestPrice(medianPrice, price) + print(nearestPrice) + d['O'] = nearestPrice + d['P'] = total + + } + } + + }) + } + //查找三方价格 + def thirdPrice(path1) { + def count = 0 + ExcelUtils.multiThreadingModify( path1, { List d -> + count++ + if (count % 10 == 0) { + println("已处理:" + count) + } + def city = "福州" + def region = d['B'] + def name = d['C'] + def valusMap = CommonDatautils.findThirdPartyPrice(city, region, name, 202505, 202508) + if (valusMap) { + def dataMap = valusMap.get(city + "##" + region + "##" + name) + if (dataMap.get("anjuke_esf_price")) { + d['F'] = (int)(dataMap.get("anjuke_esf_price") * 0.9) + } + if (dataMap.get("maitian_esf_price")) { + d['G'] = (int)(dataMap.get("maitian_esf_price") * 0.9) + } + if (dataMap.get("soufang_esf_price")) { + d['H'] = (int)(dataMap.get("soufang_esf_price") * 0.9) + } + if (dataMap.get("cityhouse_esf_price")) { + d['I'] = (int)(dataMap.get("cityhouse_esf_price") * 0.9) + } + } + + }) + } + + def priceAvg(path1) { + def count = 0 + ExcelUtils.multiThreadingModify(path1, { List d -> + count++ + if (count % 10 == 0) { + println("已处理:" + count) + } + if (d['B']) { + def city + def region + def name + if (d['B'] && d['C']) { + city ="福州" + region = d['B'] + name = d['C'] + String sql = "select * from community where city = '${city}' and region = '${region}' and name.keyword = '${name}' " + + "and source = '贝壳找房' "; + ElasticUtils.executeSqlQuery(sql, { result -> + if (result.size() > 0) { + println(name +" "+result.avg_price) + if ("avg_price" in result) { + d['N'] = result.avg_price + } + if ("half_year_deal_avg_price" in result) { + d['E'] = result.half_year_deal_avg_price + } + } + }) + } + } + + }) + } + + def futurePrice(path1) { + DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017) + def count = 0 + ExcelUtils.removeEmptyRow(path1) + ExcelUtils.defineByInputThreadingModify(1, path1, { List d -> + count++ + if (count % 10 == 0) { + println("已处理:" + count) + } + if (d['C'] && d['D'] && d['E']) { + String city = d['C'] + String region = d['D'] + String name = d['E'] + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject("weekend", -1)).limit(1) + def futurePrice + for (def db : dbcursor) { + if (db.future_value) { + futurePrice = db.future_value + d['H'] = futurePrice + break; + } + } + } + + }) + } + + //重算小区价格 + def rePrice(path2) { + ExcelUtils.removeEmptyRow(path2) + ExcelUtils.defineByInputThreadingModify(10, path2, { List d -> + if (d['A'] && d['B'] && d['C']) { + String city = d['A'] + String region = d['B'] + String name = d['C'] +// CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240721", "yyyyMMdd")) + CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240915", "yyyyMMdd")) + CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240922", "yyyyMMdd")) + CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240929", "yyyyMMdd")) + CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20241006", "yyyyMMdd")) + CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20241013", "yyyyMMdd")) + + } + + }) + } + + + // 计算中位数 + def calculateMedian(prices) { + def sortedPrices = prices.sort() + def size = sortedPrices.size() + if (size % 2 == 0) { + // 如果列表长度为偶数,则取中间两个数的平均值作为中位数 + def midIndex = (size / 2).toInteger() + return (sortedPrices[midIndex - 1] + sortedPrices[midIndex]) / 2.0 + } else { + // 如果列表长度为奇数,则取中间的数作为中位数 + return sortedPrices[((size - 1) / 2).toInteger()] + } + } + + def findNearestPrice(targetPrice, validPrices) { + def nearestPrice = validPrices.first() // 假设第一个有效价格是最接近的价格 + def minDiff = Math.abs(targetPrice - nearestPrice) + + validPrices.each { price -> + def diff = Math.abs(targetPrice - price) + if (diff < minDiff) { + minDiff = diff + nearestPrice = price + } + } + + return nearestPrice + } + + def match235(path2) { + DBCollection DATA_PROCESS = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018); + + def count = 0 + ExcelUtils.defineByInputThreadingModify(1, path2, { List d -> + count++ + if (count % 10 == 0) { + println("已处理:" + count) + } + def matchCity = d['B'] + def matchRegion = d['C'] + def matchName = d['D'] + def q = new BasicDBObject(); + q.put("match_city", matchCity) + q.put("match_region", matchRegion) + q.put("match_name", matchName) + q.put("source", "链家") + q.put("s_date", 202404) + def data = DATA_PROCESS.find(q) + for (da in data) { + if (da) { + def city = da.city + def region = da.region + def name = da.name + d['G'] = city + d['H'] = region + d['I'] = name + } + } + + + }) + } +}