This commit is contained in:
2024-01-29 15:39:40 +08:00
parent 6463280164
commit 6a56aa1825
13 changed files with 310 additions and 105 deletions
@@ -29,7 +29,7 @@ class DealPriceDataProcess {
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
task.deleteThirdPartyData()//删除第三方
task.delDealByPe()//删除成交
// task.delDealByPe()//删除成交
// task.delDealData()
// task.exportThirdParty()
@@ -37,8 +37,7 @@ class DealPriceDataProcess {
}
def delDealByPe(){
def yearMonth = 202312
def thirdMonth = 202112
def yearMonth = 202401
def rets = findAllDealByPe(yearMonth, 18)
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-12-27", "2024-01-10")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -1288,7 +1288,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20231222--修正人工指导价.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
@@ -1303,13 +1303,13 @@ ExcelUtils.multiThreadingModify(fileName, {d->
if(!d['I']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
d['I'] = dist.region
d['F'] = dist.region
// d['H'] = dist.name
d['J'] = dist.status
if(d['A']=="天门" || d['A']=="迪庆"){
def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
d['K'] = val.toInteger()
}
d['G'] = dist.status
// if(d['A']=="天门" || d['A']=="迪庆"){
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
// d['K'] = val.toInteger()
// }
}
}
}
@@ -54,7 +54,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
// ps.addBatch(id)
// }
// }
//
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年9月总量","23年10月总量","23年11月总量","占比","贝壳","链家", "我爱我家","幸福里APP","中原"]
rets << ["城市","23年10月总量","23年11月总量","23年12月总量","占比","贝壳","链家", "我爱我家","幸福里APP","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-09-01"
def date = "2023-10-01"
def endDate = "2023-11-01"
def newDate = "2023-12-01"
def startDate = "2023-10-01"
def date = "2023-11-01"
def endDate = "2023-12-01"
def newDate = "2024-01-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -17,7 +17,7 @@ class ExportPgsqlBaseEvaluation {
def database = "base"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\"
static main(args) {
@@ -374,15 +374,18 @@ class OpenApiUtils {
}
static def searchBaiDuMap(String address, String region, int cityCode) {
static def searchBaiDuMap(String region, String township, String keyword,int cityCode) {
def rets = []
keyword = keyword.replace("居民委员会","").replace("居委会","").replace("居委会","").replace("居委","").replace("村民委员会","").replace("村村委会","村").replace("村委会","村")
def address = keyword
try {
Thread.sleep(500)
long t = System.currentTimeMillis()
// String url = "http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1" +
// "&qt=con&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&sug=0&l=13" +
// "&sug_forward=&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}"
String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
// String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
def content = new HttpRequest().get(url, null, "utf-8")
JSONObject json = JSONObject.fromObject(content)
@@ -390,22 +393,46 @@ class OpenApiUtils {
for (def loc : json.content) {
String name = formatName(address)
if(loc.uid) {
JSONObject uidInfo = BaiDuApiUtils.getDetailByUid(loc.uid)
JSONObject uidInfo = null;//BaiDuApiUtils.getDetailByUid(loc.uid)
if(uidInfo != null) {
rets = createData(rets, uidInfo, loc, region, name, address)
}else{
def baiduRegion = loc.area_name.replace(loc.city_name,"")
def baiduCity = loc.city_name
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
if(!baiduCity){
baiduCity = loc.history_address
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
}
if(!baiduCity){
baiduCity = loc.area_name
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
}
def baiduRegion = loc.area_name.replace(baiduCity,"")
keyword = keyword.replace("社区","")
def bdName = loc.name.replace("居民委员会","").replace("居委会","").replace("居委","").replace("村民委员会","").replace("村村委会","村").replace("村委会","村").replace("社区","")
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(bdName, keyword);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, keyword);
if(score1 || score2){
def bean = [:]
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
bean.township = loc.admin_info.town_name
if(loc.admin_info){
bean.township = loc.admin_info.town_name
}
def locArrs = BaiduCoordinateConverter.bdMercator2Latlng(loc.diPointX/100.0, loc.diPointY/100.0)
if(locArrs){
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0], locArrs[1])
@@ -48,13 +48,13 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-11-01"
static def statDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1),"yyyy-MM-dd")
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年12月\\查询1.DBF");//友达二手房存入mongo
task.esfParser("D:\\数据\\友达\\2024年1月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
}
File diff suppressed because one or more lines are too long
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.village
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
@@ -13,43 +14,77 @@ DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
def filePath = "D:\\数据\\涉农\\县.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
// if(!row['I']){
// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
// if(rets){
// def obj = findOneByName(rets, row['C']+"人民政府")
// if(obj){
// row['I'] = obj.lng
// row['J'] = obj.lat
// println row['C'] + " "+obj
// }
// }
// }
// if(row['D'] && !row['F']){
// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
// def map = ["lon":row['D'], "lat":row['E']]
// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
// row['F'] = township
// }
//def s = getTownshipInfo("118.702988,39.740979")
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
//ExcelUtils.multiThreadingModify(filePath, { row ->
//// if(!row['I']){
//// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
//// if(rets){
//// def obj = findOneByName(rets, row['C']+"人民政府")
//// if(obj){
//// row['I'] = obj.lng
//// row['J'] = obj.lat
//// println row['C'] + " "+obj
//// }
//// }
//// }
//
//// if(row['D'] && !row['F']){
//// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
//// def map = ["lon":row['D'], "lat":row['E']]
//// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
//// row['F'] = township
//// }
//
//// if(row['F'] && !row['J']){
//// def q = new BasicDBObject()
//// q.put("city_name", row['B'])
//// q.put("area_name", row['C'])
//// def streetName = row['F']
//// q.put("street_name", streetName)
//// def count = COMMUNITY_CONN.count(q);
//// row['J'] = count
////// row['H'] = streetName
//// }
//// if(!row['H']){
//// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
//// if(gaodeMap && gaodeMap.township){
//// row['H'] = gaodeMap.township
//// }
//// }
//
//// if(row['J'] && row['J'].toString().toDouble()>0){
//// def up_query = new BasicDBObject()
//// up_query.put("city_name", row['B'])
//// up_query.put("area_name", row['C'])
//// up_query.put("street_name", row['F'])
//// def udata = new BasicDBObject()
//// udata.put("has_gov", "是")
//// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
//// }
//
//
//})
def objs = COMMUNITY_CONN.find(new BasicDBObject())
for(def d : objs){
if(d.name){
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
def udata = new BasicDBObject()
udata.put("name_pinyin", py1)
if(d.location){
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
udata.put("location", ["lat":lat, "lon":lon])
}
if(row['F'] && !row['G']){
def q = new BasicDBObject()
q.put("city_name", row['B'])
q.put("area_name", row['C'])
def streetName = row['F']
q.put("street_name", streetName)
def count = COMMUNITY_CONN.count(q);
row['G'] = count
// row['H'] = streetName
q.put("_id", d._id)
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
}
// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
// if(gaodeMap && gaodeMap.township){
// row['H'] = gaodeMap.township
// }
})
}
def getTownshipInfo(def glocation){
@@ -0,0 +1,47 @@
package com.fangjia.script.gj.village
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community_new_2023", MONGO100_URL)
filePath = "D:\\Êý¾Ý\\ɿũ\\18Íò×îÖÕ-id.xlsx"
def index = 0
ExcelUtils.multiThreadingModify(filePath, { d->
if(d['A']){
def q = new BasicDBObject("_id", d['A'])
def dbo = COMMUNITY_CONN.findOne(q)
if(dbo){
index++
println index
COMMUNITY_CONN.remove(q)
}else{
// def udata = new BasicDBObject()
// udata.put("_id" , d['A'])
// udata.put("province" , d['A'])
// udata.put("province_code" , d['A'])
// udata.put("city_code" , d['A'])
// udata.put("area_name" , d['A'])
// udata.put("area_code" , d['A'])
// udata.put("street_name" , d['A'])
// udata.put("street_code" , d['A'])
// udata.put("name" , d['A'])
// udata.put("code" , d['A'])
// udata.put("type_code" , d['A'])
// udata.put("type_name" , d['A'])
// udata.put("location_type" , d['A'])
// udata.put("geohash" , d['A'])
// udata.put("location" , d['A'])
// udata.put("new_poi_name" , d['A'])
// udata.put("new_area_name" , d['A'])
// udata.put("city_name" , d['A'])
// COMMUNITY_CONN.save(udata);
}
}
})
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.lixue.DB.City
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
@@ -44,15 +45,29 @@ class SuggestionDataSynch extends BaseBatch{
static main(args) {
runBatch(new SuggestionDataSynch(), args);
}
@Override
public void process(Object args) {
run()
// updateCode()
// getExitsCity()
}
def getExitsCity(){
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
def cities = SUGGESTION_CONN.distinct("city")
def simpleCities = []
for (def c :cities) {
def city = RegionData.formatCity(c)
simpleCities.add(city)
}
// print(simpleCities)
return simpleCities
}
@@ -63,10 +78,14 @@ class SuggestionDataSynch extends BaseBatch{
def index = 0
fullCityMap = queryFullNameCity()
mcaDistMap = queryDistNameCity()
def existCities = getExitsCity()
for (def c : citys) {
def allRets = []
if(["上海","北京","广州","深圳","成都","杭州","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳","怀化","郴州","益阳","永州","娄底","湘西","长沙","常德","株洲","岳阳","衡阳","湘潭","张家界","邵阳","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁"].contains(c)){
// if(["上海","北京","广州","深圳","成都","杭州","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳","怀化","郴州","益阳","永州","娄底","湘西","长沙","常德","株洲","岳阳","衡阳","湘潭","张家界","邵阳","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁"].contains(c)){
if(existCities.contains(c)){
continue;
}
index++
@@ -103,9 +122,9 @@ class SuggestionDataSynch extends BaseBatch{
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
count = SUGGESTION_CONN.count(q)
}
if(count>0){
SUGGESTION_CONN.remove(q)
}
// if(count>0){
// SUGGESTION_CONN.remove(q)
// }
def rets = []
for(def i=0;i<rows.size();i++){
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 3
def nThread = Runtime.getRuntime().availableProcessors() * 5
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)