Merge branch 'script_gj' into 'master'

Script gj

See merge request !748
This commit is contained in:
2023-11-13 15:33:19 +08:00
14 changed files with 784 additions and 32 deletions
@@ -0,0 +1,76 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 添加人工指导价,并重算价格
* 小付的人工指导价不能覆盖
*/
class BatchAddPersonGuiPrice {
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
static main(args) {
def task = new BatchAddPersonGuiPrice()
task.run()
}
def run(){
def path = "D:\\数据\\价格\\江苏剩下5城人工指导价录入.xlsx"
def datas = ExcelUtils.read(path, 0, 1)
for(def d : datas){
def row = findGuidingPrice(d['B'], d['C'], d['D'])
if(!row){
def obj = CommonDatautils.findDistrictById(d['A'])
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2)
//发重算均价命令
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-10','yyyy-MM-dd'))
}
}
}
def findGuidingPrice(city, region, name) {
def q = new BasicDBObject();
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("guiding_person" , "付贵生")
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd")))
def dbo = GUIDING_PRICE_COLL.findOne(q)
return dbo
}
def save(city, region, name, price, estate_type){
DBObject obj = new BasicDBObject();
obj.put("city", city);
obj.put("region", region);
obj.put("name", name);
obj.put("guiding_date", new Date()-7);
obj.put("guiding_value", price.toString().toDouble().toInteger());
obj.put("category", "district");
obj.put("guiding_type", 0);
obj.put("c_date", new Date());
obj.put("m_date", new Date());
obj.put("estate_type" , estate_type);
obj.put("house_type" , 0)
obj.put("guiding_person" , "陈杰")
GUIDING_PRICE_COLL.save(obj);
}
def addPrice(city, region, name, estateType2,startDate, endDate){
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
}
}
}
@@ -91,6 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
citys = ["上海"]
for(def city : citys){
index++
@@ -734,7 +735,7 @@ class BigdataForsaleBatch extends BaseBatch{
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate).must(QueryBuilders.termQuery("root", "lianjia.com"))
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
if(!alias){
alias = []
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-27")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-07")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -232,12 +232,11 @@ class CommonDatautils {
public static def findRegionDistrictCount(def cat, def city, def region){
DBObject q = new BasicDBObject();
q.put("visible", 0);
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
q.put("region", region);
DSCursor cursor = getDataStoreConnection().find(q);
return cursor.count();
return DbUtils.seaweed.count(q);
}
public static def findAllFiledDistrict(def cat, def city){
@@ -4002,7 +4001,7 @@ class CommonDatautils {
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
rets << ["id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
// }
})
@@ -0,0 +1,97 @@
package com.fangjia.script.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
@BatchOwner(Watcher.GaoJuan)
class DistrictFacilities2Batch extends BaseBatch {
public DBCollection onlineColl = MongoUtils.getCollectionByName("fangjia", "district_arround_poi", "mongo.db.fangjia.com")
public static void main(String[] args) {
//小区周边数据更新
this.runBatch(new DistrictFacilities2Batch(), args);
}
@Override
public void process(Object args) {
def cities = CommonDatautils.findAllCity()
for (def city : cities) {
def ret = CommonDatautils.getAllDistricts("district", city)
def index = 0;
for (def dbo : ret) {
index++;
println index + "/" + ret.size() + " " + dbo.city + " " + dbo.name
String region = dbo.get('region')
String districtName = dbo.get('name')
if (!dbo.lat2 && !dbo.lng2) {
continue;
}
def poms = findPoms(dbo.lat2 / 10000000000.0, dbo.lng2 / 10000000000.0);
DBObject object = new BasicDBObject();
object.put("hospital", unique(processType(poms.get('医院'), 1000, null, ['宠物', '动物', '口腔', '牙科', '美容', '齿科', '眼科', '号楼', '住院部', 'Pure', 'Tokushinkai', '假牙制作', '镶牙', '咪', '喵'])))
object.put("shop", processType(poms.get('超市'), 1000, ['便民商店', '便利店'], ['杂货店', '烟酒', '爱鑫', '超市便利店']))
object.put("university", processType(poms.get('学校'), 1000, ['大学'], ['食堂', '办公室', '德威', '早教', '夕阳红', '公司']))
object.put("middle_school", processType(poms.get('学校'), 1000, ['中学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
object.put("primary_school", processType(poms.get('学校'), 1000, ['小学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
object.put("kindergarten", processType(poms.get('学校'), 1000, ['幼儿园', '托儿所'], ['食堂', '亲子', '办公室', '夕阳红', '公司']))
object.put("food_shop", processType(poms.get('餐饮'), 500, ['餐饮服务']))
def market = [];
if (poms.get('商场')) {
market = market + poms.get('商场');
}
if (poms.get('超市')) {
market = market + poms.get('超市');
}
object.put("supermarket", processType(market, 1000, ['购物中心', "超市", "好又多"], ['公司', '村村寨寨', '伍缘折扣', '杂货店']))
object.put("bank", processType(poms.get('银行'), 1000, ['银行'], ['自动提款机']))
object.put("atm", processType(poms.get('ATM'), 1000, ['自动提款机', "atm", "ATM"]))
updateToDistrictFacilitys(city, region, districtName, object)
updateToDistrictFacilitysScore(city, region, districtName)
}
}
}
def findPoms(def lat, def lng) {
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
DBObject q = new BasicDBObject()
q.put('lat', new BasicDBObject(QueryOperators.LTE, map.maxLat).append(QueryOperators.GTE, map.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, map.maxLng).append(QueryOperators.GTE, map.minLng));
def cursor = POI_COLL.find(q);
def resMap = [:];
for (def dbo : cursor) {
def keyword = dbo.keyword;
def details = resMap.get(keyword);
if (!details) {
details = [];
resMap.put(keyword, details);
}
def dMap = [:];
dMap.name = dbo.name;
dMap.lat = dbo.lat;
dMap.lng = dbo.lng;
dMap.distance = com.fangjia.script.util.GeoUtils.calculateDistance(lat, lng, dbo.lat, dbo.lng);
details << dMap;
}
resMap.each { k, v ->
println "[${k}] " + v;
}
return resMap;
}
}
@@ -566,6 +566,23 @@ class OpenApiUtils {
return null;
}
static def jsHouseCase(city, region, name, propertyTypeCode){
def url = "http://localhost:8080/fangjia-open/house/case?city=${city}&district=${region}&buildingName=${name}&propertyTypeCode=${propertyTypeCode}"
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.commStat
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
@@ -27,7 +28,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
//for(def city : citys){
@@ -308,21 +309,65 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
//区域级统计
def city = "合肥"
def regions = CommonDatautils.getCityRegions(city)
def rows = []
for(def region : regions){
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
def alias_count = 0
def addr_alias_count = 0
for(def r : districts){
if(r.alias && r.alias.size()>0){
alias_count ++
//def city = "合肥"
//def regions = CommonDatautils.getCityRegions(city)
//def rows = []
//for(def region : regions){
// def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
// def alias_count = 0
// def addr_alias_count = 0
// for(def r : districts){
// if(r.alias && r.alias.size()>0){
// alias_count ++
// }
// if(r.addr_alias && r.addr_alias.size()>0){
// addr_alias_count++
// }
// }
// rows << [region, districts.size(), alias_count, addr_alias_count]
//}
//ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\区县楼盘量统计.xlsx",{d->
if(d['C'] && !d['I']){
def city = RegionData.formatCity(d['C'])
if(!city){
city = RegionData.formatCity(d['B'])
}
if(r.addr_alias && r.addr_alias.size()>0){
addr_alias_count++
if(city && d['D']){
def region = RegionData.formatRegion(city, d['D'])
if(region){
def count = CommonDatautils.findRegionDistrictCount("district", city, region)
println city +" "+ region+" "+count
d['I'] = city
d['J'] = region
d['K'] = count
}
// def dists = CommonDatautils.findAllDistricts(d['C'], "district", ["city","region","name"])
//d['B'] = dists.size()
}
}
rows << [region, districts.size(), alias_count, addr_alias_count]
}
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
// def q = new BasicDBObject()
// q.put("city",d['B'])
// q.put("district","")
// def dbo = mca_administrative_division_coll.findOne(q)
// if(!dbo){
// q.put("city", "省直辖县级行政区划")
// q.put("district", d['B'])
// dbo = mca_administrative_division_coll.findOne(q)
// }
// if(dbo){
// d['A']=dbo.province
// }
})
//def citys = ["北京","上海","重庆","天津","石家庄","唐山","秦皇岛","邯郸","邢台","保定","张家口","承德","沧州","廊坊","衡水","太原","大同","阳泉","长治","晋城","朔州","晋中","运城","忻州","临汾","吕梁","呼和浩特","包头","乌海","赤峰","通辽","鄂尔多斯","呼伦贝尔","巴彦淖尔","乌兰察布","兴安盟","锡林郭勒盟","阿拉善盟","沈阳","大连","鞍山","抚顺","本溪","丹东","锦州","营口","阜新","辽阳","盘锦","铁岭","朝阳","葫芦岛","长春","吉林","四平","辽源","通化","白山","松原","白城","延边","哈尔滨","齐齐哈尔","鸡西","鹤岗","双鸭山","大庆","伊春","佳木斯","七台河","牡丹江","黑河","绥化","大兴安岭","南京","无锡","徐州","常州","苏州","南通","连云港","淮安","盐城","扬州","镇江","泰州","宿迁","杭州","宁波","温州","嘉兴","湖州","绍兴","金华","衢州","舟山","台州","丽水","合肥","芜湖","蚌埠","淮南","马鞍山","淮北","铜陵","安庆","黄山","滁州","阜阳","宿州","六安","亳州","池州","宣城","福州","厦门","莆田","三明","泉州","漳州","南平","龙岩","宁德","南昌","景德镇","萍乡","九江","新余","鹰潭","赣州","吉安","宜春","抚州","上饶","济南","青岛","淄博","枣庄","东营","烟台","潍坊","济宁","泰安","威海","日照","临沂","德州","聊城","滨州","菏泽","郑州","开封","洛阳","平顶山","安阳","鹤壁","新乡","焦作","濮阳","许昌","漯河","三门峡","南阳","商丘","信阳","周口","驻马店","武汉","黄石","十堰","宜昌","襄阳","鄂州","荆门","孝感","荆州","黄冈","咸宁","随州","恩施","长沙","株洲","湘潭","衡阳","邵阳","岳阳","常德","张家界","益阳","郴州","永州","怀化","娄底","湘西","广州","韶关","深圳","珠海","汕头","佛山","江门","湛江","茂名","肇庆","惠州","梅州","汕尾","河源","阳江","清远","东莞","中山","潮州","揭阳","云浮","南宁","柳州","桂林","梧州","北海","防城港","钦州","贵港","玉林","百色","贺州","河池","来宾","崇左","海口","三亚","三沙","儋州","成都","自贡","攀枝花","泸州","德阳","绵阳","广元","遂宁","内江","乐山","南充","眉山","宜宾","广安","达州","雅安","巴中","资阳","阿坝州","甘孜州","凉山州","贵阳","六盘水","遵义","安顺","毕节","铜仁","黔西南","黔东南","黔南","昆明","曲靖","玉溪","保山","昭通","丽江","普洱","临沧","楚雄","红河","文山州","西双版纳州","大理","德宏州","怒江州","迪庆州","拉萨","日喀则","昌都","林芝","山南","那曲","阿里","西安","铜川","宝鸡","咸阳","渭南","延安","汉中","榆林","安康","商洛","兰州","嘉峪关","金昌","白银","天水","武威","张掖","平凉","酒泉","庆阳","定西","陇南","临夏","甘南","西宁","海东","海北州","黄南州","海南州","果洛州","玉树州","海西州","银川","石嘴山","吴忠","固原","中卫","乌鲁木齐","克拉玛依","吐鲁番","哈密","昌吉","博尔塔拉","巴音郭楞","阿克苏","克孜勒苏","喀什","和田","伊犁","塔城","阿勒泰","济源","仙桃","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","石河子","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉","昆玉"]
//
//def fjwCitys = CommonDatautils.findAllCity()
//for(def fjw_city : fjwCitys){
// if(!citys.contains(fjw_city)){
// println fjw_city
// }
//}
@@ -5,6 +5,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -22,12 +23,16 @@ class ApiTest {
static main(args) {
def task = new ApiTest()
// task.rankRunTest()
// task.similarRunTest()
task.loupanLevelTest()
// task.rankRunTest()//小区排名接口测试
// task.similarRunTest()//相似房源接口测试
// task.loupanLevelTest()//楼盘评级接口测试
// task.jsAbcHouseCastTest()//江苏农总行房源接口测试
task.arroundFacilities()//周边poi
}
def loupanLevelTest(){
ExcelUtils.modify("D:\\数据\\卓信\\上海 - 副本.xlsx", { d ->
if (d['A']) {
@@ -125,6 +130,150 @@ class ApiTest {
return dbo
}
def jsAbcHouseCastTest(){
def citys = ["无锡"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","estate_type2"])
def rets = getHouseCases(districts)
ExcelUtils.write("D:\\数据\\江苏\\房源样本.xlsx", rets, city)
}
}
def getHouseCases(districts){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
rets << ["城市","区域","名称", "物业类型", "最小案例","最大案例", "案例"]
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def propertyTypeCode = null
if(d.estate_type2=="普通住宅"){
propertyTypeCode = "01"
}else if(d.estate_type2=="别墅"){
propertyTypeCode = "02"
}else if(d.estate_type2=="商住"){
propertyTypeCode = "03"
}
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCase(d.city, d.region, d.name, propertyTypeCode)
if(results){
def row = []
row << d.city
row << d.region
row << d.name
row << d.estate_type2
row << results.minUnitPrice
row << results.maxUnitPrice
def datas = results.datas
if(datas){
def str = ""
for(def r : datas){
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
}
row << str
}
rets << row
}
println totalCount.get()+" "+d.city +" "+d.region+" "+d.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def arroundFacilities(){
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city","region","name","estate_type2"])
// districts = districts[0..1000]
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
rets << ["城市","区域","名称", "重点学校","学校", "公交","地铁","超市","医院","银行"]
for(def i=0;i<districts.size();i++){
final def d = districts[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def row = []
row << d.city
row << d.region
row << d.name
def facilityObj = OpenApiClient.facilitys(d.city, d.region, d.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(result.primarySchool){
row <<result.primarySchool.join(",").replace('"',"")
}else {
row << ""
}
if(result.school){
row << result.school.join(",").replace('"',"")
}else {
row << ""
}
if(result.bus){
row << result.bus.join(",").replace('"',"")
}else{
row << ""
}
if(result.metro){
row << result.metro.join(",").replace('"',"")
}else{
row << ""
}
if(result.supermarket){
row << result.supermarket.join(",").replace('"',"")
}else{
row << ""
}
if(result.hospital){
row << result.hospital.join(",").replace('"',"")
}else{
row << ""
}
if(result.bank){
row << result.bank.join(",").replace('"',"")
}else{
row << ""
}
}
println totalCount.get() + " "+d.region +" "+d.name
rets << row
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
ExcelUtils.write("D:\\数据\\江苏\\南京周边配套.xlsx", rets, "南京")
}
}
@@ -21,7 +21,8 @@ class MsbPocTest {
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
// test("D:\\数据\\美术宝\\test2.xlsx")
cityPrice()
// cityPrice()
mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
}
def cityPrice(){
@@ -130,6 +131,44 @@ class MsbPocTest {
})
}
def mergeMatchPrice(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['L']){
def city = row['C']
def region = row['D']
def name = row['F']
def address = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, "property", false)
if(jsonObject && !jsonObject.containsKey("result")){
jsonObject = FangjiaClientUtils.newMatchPrice(city, address, "property", false)
}
if(jsonObject && !jsonObject.containsKey("result")){
jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, null, true)
}
if(jsonObject && !jsonObject.containsKey("result")){
jsonObject = FangjiaClientUtils.newMatchPrice(city, address, null, true)
}
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
row['L'] = resultMap.get("city")
row['M'] = resultMap.get("district")
row['N'] = resultMap.get("name")
def type = resultMap.get("propertyType")
if(type && type!='null'){
row['O'] = type
}
if(resultMap.get("avgPrice")){
row['P'] = resultMap.get("avgPrice")
}
}
}
})
}
def addPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
@@ -0,0 +1,286 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.*
import java.util.concurrent.atomic.AtomicLong
class SuggestionDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
public static def provinceMap = CommonDatautils.findAllCityProvince()
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
public static def priceMap = null
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
def keys = new CopyOnWriteArraySet();
def fullCityMap = null
def mcaDistMap = null
static main(args) {
runBatch(new SuggestionDataSynch(), args);
}
@Override
public void process(Object args) {
run()
}
def run() {
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
def citys = CommonDatautils.findAllCity()
def index = 0
fullCityMap = queryFullNameCity()
mcaDistMap = queryDistNameCity()
for (def c : citys) {
index++
if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
updating(c, "district")
updating(c, "office")
updating(c, "shop")
println index + " " + c + " 完成"
}
// poiData(c)
}
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
def key = dist.city+"#"+dist.region+"#"+dist.name
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!keys.contains(key)){
keys << key
def map = createData(dist)
//往100 上同步
updateData(map)
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
}
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(dist){
def map = [:]
map.id = dist._id.toString()
def province = provinceMap.get(dist.city)
if(province){
map.province = ProvinceData.getFullProvince(province)
}
def data = fullCityMap.get(dist.city)
map.city = data.city
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
if(distData){
map.region = distData.district
}else{
map.region = dist.region
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
}
map.name = dist.name
map.address = dist.address
if(dist.alias && dist.alias.size()>0){
map.alias = dist.alias
}
if(dist.addr_alias && dist.addr_alias.size()>0){
map.addr_alias = dist.addr_alias
}
map.type = dist.cat
if(dist.lng){
def loc = [:]
loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0)
map.location = loc
}
map.source = "FJW"
return map
}
def findAllDistrictSellPrice(){
def q = new BasicDBObject()
q.put("date", lastMonthDate)
def priceMap = [:]
DISTRICT_SELL_STAT.find(q).each{
priceMap.put(it.commid, it)
}
return priceMap
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
if(k!="id"){
udata.put(k,v)
}
}
SUGGESTION_CONN.update(new BasicDBObject('_id', map.id), new BasicDBObject('$set', udata), true, false)
}
def poiData(city){
def data = fullCityMap.get(city)
if(data){
def results = CommonDatautils.findAmapDetailBySql(data.city)
createPoi(results, data)
println city +" 完成!"
}
}
def createPoi(results, data){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
for(def i=0;i<results.size();i++){
final def dist = results[i]
def key = dist.city+"#"+dist.district+"#"+dist.name
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!keys.contains(key)){
keys << key
def map = [:]
map.id = dist.id
map.province = dist.province
map.city = dist.city
map.region = dist.district
map.name = dist.name
map.address = dist.address
map.type = "poi"
map.cat = dist.cat
map.location = dist.location
map.source = "FJW"
//往100 上同步
updateData(map)
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
}
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
def fjwcity = null
if(d.level == "城市"){
fjwcity = RegionData.formatCity(d.city)
if(!fjwcity){
if(d.city == "市辖区"){
fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
d.city = d.province
}
}
}
}else{
if(d.city == "省直辖县级行政区划" && d.district){
fjwcity = RegionData.formatCity(d.district)
if(fjwcity){
d.city = d.district
}
}
}
if(fjwcity){
retMap.put(fjwcity, d)
}else{
println d.toString() +" 格式化不成功!"
}
}
return retMap;
}
def queryDistNameCity(){
def q = new BasicDBObject()
q.put("level","区域")
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
if(d.city == "省直辖县级行政区划"){
continue
}else if(d.city == "市辖区"){
def fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}else{
def fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}
}
return retMap;
}
}
@@ -765,9 +765,8 @@ public class OpenApiClient {
params.put("region", region)
params.put("name", name)
try {
String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8")
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
@@ -23,6 +23,7 @@ public class ProvinceData {
put("辽宁", ["辽宁省", "辽"]);
put("黑龙江", ["黑龙江省", "黑"]);
put("湖南", ["湖南省", "湘"]);
put("湖北", ["湖北省", "鄂"]);
put("安徽", ["安徽省", "皖"]);
put("山东", ["山东省", "鲁"]);
put("新疆", ["新疆维吾尔自治区", "新"]);
@@ -62,6 +63,38 @@ public class ProvinceData {
public static Map<String, String> CITY_PROVINCE_MAP = new HashMap<String, String>();
static DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com", 27017);
static final Map<String, String> PROVINCE_MAP = new HashMap<String, String>() {{
put("河北", "河北省");
put("河南", "河南省");
put("云南", "云南省");
put("辽宁", "辽宁省");
put("黑龙江", "黑龙江省");
put("湖南", "湖南省");
put("湖北", "湖北省");
put("安徽", "安徽省");
put("山东", "山东省");
put("新疆", "新疆维吾尔自治区");
put("江苏", "江苏省");
put("浙江", "浙江省");
put("江西", "江西省");
put("广西", "广西壮族自治区");
put("甘肃", "甘肃省");
put("山西", "山西省");
put("内蒙古", "内蒙古自治区");
put("陕西", "陕西省");
put("吉林", "吉林省");
put("福建", "福建省");
put("贵州", "贵州省");
put("广东", "广东省");
put("青海", "青海省");
put("西藏", "西藏自治区");
put("四川", "四川省");
put("宁夏", "宁夏回族自治区");
put("海南", "海南省");
put("台湾", "台湾省");
}};
static {
logger.info("开始加载省份信息");
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("province", 1).append("display", 1)).batchSize(300);
@@ -109,6 +142,16 @@ public class ProvinceData {
return CITY_PROVINCE_MAP.get(city);
}
/**
* 根据省的简称获取省的全称
* @param province
* @return
*/
public static String getFullProvince(String province){
return PROVINCE_MAP.get(province);
}
/**
* 获取省份下的所有城市
* @param province
@@ -11,6 +11,7 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def fileStartDate = DateUtil.parseDate("20230201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230301", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
@@ -38,7 +39,7 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
ExcelUtils.write("D:\\亲邻数据${fileStartDate.format('yyyyMM')}.xlsx", data, "查询日志")
//def list = []
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20231001", "yyyyMMdd")
def startDate = DateUtil.parseDate("20231001", "yyyyMMdd")
def endDate = DateUtil.parseDate("20231101", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -39,6 +39,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202310.xlsx", data, "楼盘评情")
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202311.xlsx", data, "楼盘评情")