Merge branch 'script_gj' into 'master'

Script gj

See merge request !1177
This commit is contained in:
2025-09-28 09:04:38 +08:00
14 changed files with 801 additions and 50 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份成交数据.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳8月成交数据.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -77,14 +77,14 @@ for(def d : datas){
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].toString().toDouble())
if(d['I']){
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
map.sell_price = (int)(d['I'].toString().replace("万","").toDouble()*10000)
}
// map.trade_date = d['J']
def trade_date = d['J']
if(trade_date){
map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
}
map.trade_date = d['J']
// def trade_date = d['J']
// if(trade_date){
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
// }
// else{
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
@@ -825,10 +825,31 @@ class CommonDatautils {
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
if(!d.township){
rets << d
rets << d
}
return rets;
}
public static def findAllRegionDistricts(def city, def region, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city)
q.put("region", region)
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
@@ -4056,6 +4077,9 @@ class CommonDatautils {
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
// rowMap.location = seawater.location
rowMap.title = seawater.name
rets << rowMap
}
}
@@ -4844,6 +4868,7 @@ class CommonDatautils {
query.append("cat", "district")
query.append("city", city)
query.append("visible", 0)
query.append("name", "长城大厦")
DBObject sdq = new BasicDBObject();
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
@@ -4905,6 +4930,10 @@ class CommonDatautils {
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if (!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price > 3000) {
dataMap.put("soufang_esf_price", db.soufang_esf_price)
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
@@ -1,19 +1,12 @@
package com.fangjia.script.fuzhou
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import com.mongodb.*
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
/**
* 福州数据结果统计
*/
@@ -34,6 +27,20 @@ class FzDataResultStat {
//// task.margeExcel()
// task.saveCsvData()
// task.testNewPrice()
// task.statHouse()
}
def statHouse(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\福州待调查小区.xlsx",{d->
def o = CommonDatautils.findDistrictById(d['A'])
if(o){
def count = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20220818)
d['K'] = count
println o.name +" "+count
}
})
}
def margeExcel(){
@@ -89,7 +96,7 @@ class FzDataResultStat {
map2.put(key, obj)
}
def rows = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
rows << ["ID","城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
def index = 0
map1.each {k,obj1->
def keys = k.split("##")
@@ -114,7 +121,7 @@ class FzDataResultStat {
if(markup){
markup_abs = Math.abs(markup)
}
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
def row = [obj1._id.toString(), obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
rows << row
index++
println row
@@ -0,0 +1,500 @@
package com.fangjia.script.majintao.FuZhou
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.atomic.AtomicInteger
class GongShu {
static main(args) {
def task = new GongShu();
task.run();
}
def run() {
String path1 = "D:\\数据\\福州\\福州200个小区抽样价格核验202509.xlsx";
// String path1 = "D:\\数据\\福州\\价格比对_20240915.xlsx";
String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
// String path2 = "E:\\goojia\\file\\excel文件\\福州\\福州202410小区价格.xlsx";
String path3 = "E:\\goojia\\file\\excel文件\\福州\\福州贝壳小区.xlsx";
// String path2 = "E:\\goojia\\file\\excel文件\\福州\\1.xlsx";
// String path1 = "E:\\goojia\\file\\excel文件\\福州\\福州全部小区.xlsx";
// findID(path2)
// findDelDistrict(path2)
// findDistrictByAlias(path2)
// match19UseMAP(path2)
// addFjwPrice(path2)
// match(path2, path1)
// match2(path3, path1)
// houseCase(path2)
thirdPrice(path1)
// priceAvg(path1)
// futurePrice(path2)
// rePrice(path2)
}
def findID(path2) {
def count = new AtomicInteger(0)
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def city = d['C']
def region = d['D']
def name = d['E']
def district = CommonDatautils.findDistrict(city, region, name, "district")
if (district) {
d['B'] = district._id.toString()
}
})
}
def findDelDistrict(path2) {
def count = new AtomicInteger(0)
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def city = d['C']
def region = d['D']
def name = d['E']
def district = CommonDatautils.findDistrictAll(city, region, name, "district")
if ("1" == district.status.toString()) {
// d['B'] = district._id.toString()
d['G'] = "已删除"
}
})
}
def findDistrictByAlias(path2) {
def count = new AtomicInteger(0)
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
if (d['G']) {
def city = d['C']
def region = d['D']
def name = d['E']
def district = CommonDatautils.findSeaweedByAlias(city, region, name, "district")
if (district) {
d['H'] = district.get(0)._id.toString()
}
}
})
}
def match19UseMAP(path1) {
def count = new AtomicInteger(0)
ExcelUtils.defineByInputThreadingModify(10, path1, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def address = d['B'] + d['D'] + d['C']
def city = d['B']
def name = d['C']
def region = d['D']
//19匹配
String url = "http://192.168.0.19:31080/FangJiaNlpService/match";
def map = [:]
map.put("address", address)
map.put("city", city)
map.put("useMap", "true")
map.put("category", "property")
HttpRequest httpRequest = new HttpRequest();
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content
if (result != null && "name" in result) {
if ("city" in result) {
d['G'] = result.get("city").toString().replaceAll("null", "")
}
if ("district" in result) {
d['H'] = result.get("district").toString().replaceAll("null", "")
}
if ("name" in result) {
d['I'] = result.get("name").toString().replaceAll("null", "")
}
} else {
address = d['B'] + d['D'] + d['A']
map.put("address", address)
def content2 = httpRequest.getJsonContent(url, map, "UTF-8")
def result2 = content2
if (result2 != null && "name" in result2) {
if ("city" in result2) {
d['G'] = result2.get("city").toString().replaceAll("null", "")
}
if ("district" in result2) {
d['H'] = result2.get("district").toString().replaceAll("null", "")
}
if ("name" in result2) {
d['I'] = result2.get("name").toString().replaceAll("null", "")
}
}
}
})
}
def addFjwPrice(path1) {
ExcelUtils.defineByInputThreadingModify(5, path1, { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if (dist) {
def price = CommonDatautils.queryThisMonthDistrictAvgPrice(dist, "2024-11-01")
if (price) {
d['F'] = price.avg_price
d['G'] = price.avg_price_2b
def dbo = price
// 计算方式
if (dbo.value_type == 1) {
dbo.cal_type = "指导价"
} else if (dbo.value_type == 2) {
dbo.cal_type = "自身样本"
} else if (dbo.value_type == 3) {
dbo.cal_type = "第三方"
} else if (dbo.value_type == 4) {
dbo.cal_type = "交易数据"
} else if (dbo.value_type == 5) {
dbo.cal_type = "周边楼盘"
} else if (dbo.value_type == 6) {
dbo.cal_type = "推导"
} else if (dbo.value_type == 9) {
dbo.cal_type = "神经网络"
} else if (dbo.value_type == 10) {
dbo.cal_type = "无方式"
} else {
dbo.cal_type = ""
}
d['I'] = dbo.cal_type
}
}
}
})
}
def match2(path2, path1) {
def count = new AtomicInteger(0)
def map = [:]
ExcelUtils.removeEmptyRow(path1)
ExcelUtils.removeEmptyRow(path2)
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def key = d['G'] + d['H'] + d['I']
map.put(key, d)
})
print(map)
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
def key = "福州" + d['C'] + d['D']
if (map.containsKey(key)) {
def value = map.get(key)
d['J'] = value['B']
d['K'] = value['D']
d['L'] = value['C']
}
})
}
def match(path2, path1) {
def count = new AtomicInteger(0)
def map = [:]
ExcelUtils.removeEmptyRow(path1)
ExcelUtils.removeEmptyRow(path2)
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def key = d['C'] + d['D'] + d['E']
map.put(key, d)
})
print(map)
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
def key = "福州" + d['C'] + d['D']
if (map.containsKey(key)) {
def value = map.get(key)
if (!d['J']) {
d['J'] = value['J']
d['K'] = value['K']
d['L'] = value['L']
if (value['J']) {
d['M'] = "pp"
}
}
}
})
}
// 房源及数量
def houseCase(path2) {
def count = new AtomicInteger()
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
count.incrementAndGet()
if (count % 10 == 0) {
println("已处理:" + count)
}
def price = []
def city
def region
def name
if (d['J'] && d['K'] && d['L']) {
city = d['J']
region = d['K']
name = d['L']
def must = []
def term1 = ["terms": ["root": ["ke.com"]]] // 修正terms查询的格式
must << term1
def term2 = ["terms": ["cat": ["sell"]]]
must << term2
def term3 = ["terms": ["city": [city]]]
must << term3
def term4 = ["terms": ["region": [region]]]
must << term4
def term5 = ["terms": ["district_name": [name]]]
must << term5
def params = ["query": ["bool": ["must": must]]]
// 添加 p_date 大于 20240101 的条件
def term6 = ["range": ["p_date": ["gt": "20240401"]]]
must << term6
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/seawater-20*/_search");
def total = 0
if (s) {
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if (hits) {
total = hits.total.value
if (hits.hits.size() > 0) {
hits.hits.each { hit ->
if (hit._source && hit._source.avg_price) {
price.add(hit._source.avg_price)
}
}
}
}
}
if (price.size() > 0) {
println(price)
// 计算中位数
def medianPrice = calculateMedian(price)
// 找到与中位数最接近的有效价格
def nearestPrice = findNearestPrice(medianPrice, price)
print(nearestPrice)
d['O'] = nearestPrice
d['P'] = total
}
}
})
}
//查找三方价格
def thirdPrice(path1) {
def count = 0
ExcelUtils.multiThreadingModify( path1, { List d ->
count++
if (count % 10 == 0) {
println("已处理:" + count)
}
def city = "福州"
def region = d['B']
def name = d['C']
def valusMap = CommonDatautils.findThirdPartyPrice(city, region, name, 202505, 202508)
if (valusMap) {
def dataMap = valusMap.get(city + "##" + region + "##" + name)
if (dataMap.get("anjuke_esf_price")) {
d['F'] = (int)(dataMap.get("anjuke_esf_price") * 0.9)
}
if (dataMap.get("maitian_esf_price")) {
d['G'] = (int)(dataMap.get("maitian_esf_price") * 0.9)
}
if (dataMap.get("soufang_esf_price")) {
d['H'] = (int)(dataMap.get("soufang_esf_price") * 0.9)
}
if (dataMap.get("cityhouse_esf_price")) {
d['I'] = (int)(dataMap.get("cityhouse_esf_price") * 0.9)
}
}
})
}
def priceAvg(path1) {
def count = 0
ExcelUtils.multiThreadingModify(path1, { List d ->
count++
if (count % 10 == 0) {
println("已处理:" + count)
}
if (d['B']) {
def city
def region
def name
if (d['B'] && d['C']) {
city ="福州"
region = d['B']
name = d['C']
String sql = "select * from community where city = '${city}' and region = '${region}' and name.keyword = '${name}' " +
"and source = '贝壳找房' ";
ElasticUtils.executeSqlQuery(sql, { result ->
if (result.size() > 0) {
println(name +" "+result.avg_price)
if ("avg_price" in result) {
d['N'] = result.avg_price
}
if ("half_year_deal_avg_price" in result) {
d['E'] = result.half_year_deal_avg_price
}
}
})
}
}
})
}
def futurePrice(path1) {
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
def count = 0
ExcelUtils.removeEmptyRow(path1)
ExcelUtils.defineByInputThreadingModify(1, path1, { List d ->
count++
if (count % 10 == 0) {
println("已处理:" + count)
}
if (d['C'] && d['D'] && d['E']) {
String city = d['C']
String region = d['D']
String name = d['E']
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject("weekend", -1)).limit(1)
def futurePrice
for (def db : dbcursor) {
if (db.future_value) {
futurePrice = db.future_value
d['H'] = futurePrice
break;
}
}
}
})
}
//重算小区价格
def rePrice(path2) {
ExcelUtils.removeEmptyRow(path2)
ExcelUtils.defineByInputThreadingModify(10, path2, { List d ->
if (d['A'] && d['B'] && d['C']) {
String city = d['A']
String region = d['B']
String name = d['C']
// CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240721", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240915", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240922", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20240929", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20241006", "yyyyMMdd"))
CommonUtils.recalculationAverage(city, region, name, "小区", DateUtil.parseDate("20241013", "yyyyMMdd"))
}
})
}
// 计算中位数
def calculateMedian(prices) {
def sortedPrices = prices.sort()
def size = sortedPrices.size()
if (size % 2 == 0) {
// 如果列表长度为偶数,则取中间两个数的平均值作为中位数
def midIndex = (size / 2).toInteger()
return (sortedPrices[midIndex - 1] + sortedPrices[midIndex]) / 2.0
} else {
// 如果列表长度为奇数,则取中间的数作为中位数
return sortedPrices[((size - 1) / 2).toInteger()]
}
}
def findNearestPrice(targetPrice, validPrices) {
def nearestPrice = validPrices.first() // 假设第一个有效价格是最接近的价格
def minDiff = Math.abs(targetPrice - nearestPrice)
validPrices.each { price ->
def diff = Math.abs(targetPrice - price)
if (diff < minDiff) {
minDiff = diff
nearestPrice = price
}
}
return nearestPrice
}
def match235(path2) {
DBCollection DATA_PROCESS = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018);
def count = 0
ExcelUtils.defineByInputThreadingModify(1, path2, { List d ->
count++
if (count % 10 == 0) {
println("已处理:" + count)
}
def matchCity = d['B']
def matchRegion = d['C']
def matchName = d['D']
def q = new BasicDBObject();
q.put("match_city", matchCity)
q.put("match_region", matchRegion)
q.put("match_name", matchName)
q.put("source", "链家")
q.put("s_date", 202404)
def data = DATA_PROCESS.find(q)
for (da in data) {
if (da) {
def city = da.city
def region = da.region
def name = da.name
d['G'] = city
d['H'] = region
d['I'] = name
}
}
})
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202506.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202508.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -778,6 +778,26 @@ class OpenApiUtils {
return null;
}
static def addrVerifyGuangda(city, address){
address = URLEncoder.encode(address, "utf-8");
def url = "http://192.168.0.19:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(city){
url = url + "&city=${city}"
}
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.msgCode ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
// 省、市、区、街道、社区居委、POI补全
static def complete(city, address){
def url = "http://open.fangjia.com/addrSvc/complete?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
@@ -33,17 +33,16 @@ class SuzouPocTest {
def test(){
ExcelUtils.modify("D:\\数据\\上海联合产权交易所\\第1批拟处置房产清单(长城大厦、台山碧桂园).xlsx", { d ->
if(d['O']){
if(!d['P'] && d['I']){
def floorAdds = d['I'].split("/")
def ret1 = OpenApiClient.addressEvaluation(d['O'], null, d['D'], d['D'], d['K'], floorAdds[0], floorAdds[1])
def ret1 = OpenApiClient.addressEvaluation(d['C'], null, d['D'], d['D'], d['K'], floorAdds[0], floorAdds[1])
if (ret1) {
d['N'] = ret1.totalPrice
d['M'] = ret1.avgPrice
d['P'] = ret1.district
d['Q'] = ret1.name
d['S'] = ret1.totalPrice
d['T'] = ret1.avgPrice
d['U'] = ret1.district
d['V'] = ret1.name
}
}
});
}
@@ -12,6 +12,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import com.fangjia.script.gj.*
import org.bson.types.ObjectId
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -30,7 +31,18 @@ class DataMatch {
// println task.findAddress("昌州街道棠香路")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx")
// task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx")
task.delRegionCode("D:\\数据\\需删除区域编码.xlsx")
}
def delRegionCode(path){
ExcelUtils.modify(path,{d->
def q = new BasicDBObject("_id", new ObjectId(d['A']))
def obj = FANGJIA_OFFICIAL_AREA_MAP.findOne(q)
if(obj){
FANGJIA_OFFICIAL_AREA_MAP.remove(q)
}
})
}
def fzWeiyuanshuMatch(fileName){
@@ -189,7 +189,7 @@ class AddDevShapeData {
def run(){
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\0812'
def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\0903'
def dir = new File(directoryPath)
if (dir.isDirectory()) {
dir.eachFileRecurse(FileType.FILES) { file ->
@@ -1,10 +1,14 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -27,20 +31,21 @@ class AutoAddDistrictBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
def fullCityMap = null//CommonDatautils.queryFullNameCity()
def mcaDistMap = null//CommonDatautils.queryDistNameCityByMapping()
static main(args) {
runBatch(new AutoAddDistrictBatch(), args)
// runBatch(new AutoAddDistrictBatch(), args)
// def task = new AutoAddDistrictBatch();
def task = new AutoAddDistrictBatch();
// task.checkMap()
task.addDistrict()
}
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = ["三明"]//CommonDatautils.findAllCity()
def citys = ["泉州"]//CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "百度类型","高德地图", "高德类型"]
for(def c : citys){
@@ -52,7 +57,7 @@ class AutoAddDistrictBatch extends BaseBatch{
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\add_dist_info_1.xlsx"
fileName = "D:\\数据\\数据建设\\小区\\泉州待新增小区.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
@@ -109,7 +114,7 @@ class AutoAddDistrictBatch extends BaseBatch{
gaode_name = gaode_obj.name
}
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_type, gaode_count, gaode_type, gaode_name]
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, baidu_type, gaode_count, gaode_type, gaode_name]
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -187,6 +192,7 @@ class AutoAddDistrictBatch extends BaseBatch{
q.put("match_id", new BasicDBObject('$exists',false))
q.put("old_match_id", new BasicDBObject('$exists',false))
q.put("city", c)
q.put("district", "丰泽")
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
def rMap = [:]
for(def d : dbcursor){
@@ -293,4 +299,110 @@ class AutoAddDistrictBatch extends BaseBatch{
// d['O'] = score
})
}
def addDistrict(){
def path = "D:\\数据\\数据建设\\小区\\新增价格.xlsx"
ExcelUtils.modify(path, { d ->
def dbo = new BasicDBObject()
dbo.city = d['A'];
dbo.region = d['B'];
if(d['M']){
dbo.name = d['M']
def alias = []
alias << d['C']
dbo.alias = alias
}else{
dbo.name = d['C']
}
if(d['D']){
dbo.estate_type2 = d['D']
}else{
if(d['C'].contains("别墅")){
dbo.estate_type2 = "别墅"
}else if(d['C'].contains("商住")){
dbo.estate_type2 = "商住"
}else{
dbo.estate_type2 = "普通住宅"
}
}
dbo.status = 0;
dbo.cat = "district"
dbo.c_date = new Date()
dbo.m_date = new Date()
// def rets = getEsDatas(d['A'], d['B'], d['C'])
// for(def r : rets){
// if(r.address && !dbo.address){
// dbo.address = r.address
// }
// if(!dbo.lng){
// if(r.location){
// def lng = r.location.x
// def lat = r.location.y
// Long gaode_lng = lng*10000000000L
// Long gaode_lat = lat*10000000000L
// dbo.put("lng", gaode_lng)
// dbo.put("lat", gaode_lat)
// def bdpoint = FangjiaClientUtils.convert(lng+","+lat, 1)
// if(bdpoint){
// dbo.put("lng2", (bdpoint[0]*10000000000).longValue())
// dbo.put("lat2", (bdpoint[1]*10000000000).longValue())
// }
// }
// }
// }
// println dbo
// def q = new BasicDBObject()
// q.put("city", dbo.city)
// q.put("name", dbo.name)
// q.put("cat", "district")
// q.put("status", 1)
// q.put("region", dbo.region)
// def obj = DbUtils.seaweed.findOne(q)
// if(obj){
// println "【存在】"+dbo
// def udata = new BasicDBObject()
// udata.put("status", 1)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
// }else{
// DbUtils.seaweed.save(dbo);
//
// }
def stop = false
// while (!stop){
def dist = CommonDatautils.findDistrict(dbo.city, dbo.region, dbo.name, "district")
if(dist){
d['N'] = "已处理"
//发重算均价命令
addPrice(dist.city, dist.region, dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2025-08-01','yyyy-MM-dd'),new Date())
stop = true
}
// }
})
}
def getEsDatas(city, region, name){
String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' "
def datas = []
ElasticUtils.executeSqlQuery(sql,{ data->
datas << data
})
return datas;
}
def addPrice(city, region, name, estateType2,startDate, endDate){
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
}
}
}
@@ -43,9 +43,9 @@ class BusinessQuarterProcess {
}
def run(){
// synchBusinessData()
// updateBusinessQuarter()
updateBusinessQuarter()
// addRegion()
synchBusinessToEs()
// synchBusinessToEs()
}
def addRegion(){
@@ -172,11 +172,11 @@ class BusinessQuarterProcess {
bk_business_coll.update(new BasicDBObject("_id", row._id), new BasicDBObject('$set', row),true, false)
}
def updateBusinessQuarter(){
def citys = CommonDatautils.findAllCity()
def citys = ["上海"]//CommonDatautils.findAllCity()
for(def c : citys){
if(["上海","北京"].contains(c)){
continue
}
// if(["上海","北京"].contains(c)){
// continue
// }
def cityBusinessList = CommonDatautils.findBusinessByCity(c)
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name", "business_quarter", "lat", "lng"])
businessProcess(c, districts, cityBusinessList)
@@ -33,10 +33,10 @@ class ApiTest {
// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
// task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
// task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
//五级地址标准化
// task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
@@ -85,7 +85,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
public void process(Object args) {
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(["all"])
def citys = CcbCommonUtils.getRunningCitys(["all"])
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -201,7 +201,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
} finally {
latch.countDown()
}
};
};
})
}
latch.await()
@@ -492,7 +492,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
// def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
@@ -530,11 +531,15 @@ class FjwBigdataForsaleBatch extends BaseBatch{
}
}
if(estate_type2 && estate_type2=="±ðÊû"){
if(estate_type2 && estate_type2=="±ðÊû" && !name.contains("±ðÊû")){
def newName = name + "(别墅)"
if(!alias.contains(newName)){
alias << newName
}
newName = name + "±ðÊû"
if(!alias.contains(newName)){
alias << newName
}
}
if(estate_type2 && estate_type2=="商住"){
@@ -544,6 +549,21 @@ class FjwBigdataForsaleBatch extends BaseBatch{
}
}
if(name.contains("±ðÊû")){
def newName = name.replace("±ðÊû", "").replace("()", "").replace("£¨£©", "")
def obj = CommonDatautils.findDistrict(city,region,newName,"district")
if(!alias.contains(newName)){
alias << newName
}
if(obj && obj.alias){
for(def s1 : alias){
if(!alias.contains(s1)){
alias << s1
}
}
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
@@ -586,6 +606,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
if(vallaFlag){
continue;
}
}else if (estate_type2 && estate_type2=="±ðÊû"){
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(!vallaFlag){
continue;
}
}
@@ -628,7 +653,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
if(seawater.toilet && (seawater.toilet>=10 || seawater.toilet<1)){
continue;
}
@@ -0,0 +1,47 @@
package com.fangjia.script.gj.village
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.ExcelUtils
class GuangDaVillage {
static main(args) {
def task = new GuangDaVillage();
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\光大\\公开数据信息补充v3.xlsx",{ d->
if(!d['R']){
def dto = OpenApiUtils.addrVerifyGuangda(null, d['A'])
if(dto){
// def dto = jsonObj.result
d['F'] = dto.province
d['G'] = dto.provinceCode
d['H'] = dto.city
d['I'] = dto.cityCode;
d['J'] = dto.district
d['K'] = dto.districtCode
d['L'] = dto.town
d['M'] = dto.townCode
d['N'] = dto.committee
d['O'] = dto.committeeCode
d['P'] = dto.typeCode
d['Q'] = dto.typeName
d['R'] = dto.verifyFlag//涉农标识
d['S'] = dto.addrVerify//地址是否有效信息
d['T'] = dto.countyLevelCity//是否为县级市
d['U'] = dto.countyGov//是否为县政府
d['V'] = dto.addrArea //所属区域 农村区域 城市区域 县城城区(对公)
d['W'] = dto.devDistrict//是否为开发区
d['X'] = dto.devDistrictName//开发区
d['Y'] = dto.devLevel//开发区等级
d['Z']= dto.administrativeTowns;//是否市辖建制
d['AA'] = dto.prefectureLevelOrAbove ;// 是否为地级市以上行政区划
d['AB'] = dto.countyGovTown; //县政府所在镇
}
}
})
}
}