Merge branch 'script_gj' into 'master'
Script gj See merge request !748
This commit is contained in:
+76
@@ -0,0 +1,76 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* 添加人工指导价,并重算价格
|
||||
* 小付的人工指导价不能覆盖
|
||||
*/
|
||||
class BatchAddPersonGuiPrice {
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new BatchAddPersonGuiPrice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def path = "D:\\数据\\价格\\江苏剩下5城人工指导价录入.xlsx"
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
for(def d : datas){
|
||||
def row = findGuidingPrice(d['B'], d['C'], d['D'])
|
||||
if(!row){
|
||||
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2)
|
||||
//发重算均价命令
|
||||
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-10','yyyy-MM-dd'))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findGuidingPrice(city, region, name) {
|
||||
def q = new BasicDBObject();
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("guiding_person" , "付贵生")
|
||||
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd")))
|
||||
def dbo = GUIDING_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def save(city, region, name, price, estate_type){
|
||||
DBObject obj = new BasicDBObject();
|
||||
obj.put("city", city);
|
||||
obj.put("region", region);
|
||||
obj.put("name", name);
|
||||
obj.put("guiding_date", new Date()-7);
|
||||
obj.put("guiding_value", price.toString().toDouble().toInteger());
|
||||
obj.put("category", "district");
|
||||
obj.put("guiding_type", 0);
|
||||
obj.put("c_date", new Date());
|
||||
obj.put("m_date", new Date());
|
||||
obj.put("estate_type" , estate_type);
|
||||
obj.put("house_type" , 0)
|
||||
obj.put("guiding_person" , "陈杰")
|
||||
GUIDING_PRICE_COLL.save(obj);
|
||||
}
|
||||
|
||||
|
||||
def addPrice(city, region, name, estateType2,startDate, endDate){
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+2
-1
@@ -91,6 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
|
||||
def houseIdsfilter = findAllHouseIdsByCity()
|
||||
citys = ["上海"]
|
||||
|
||||
for(def city : citys){
|
||||
index++
|
||||
@@ -734,7 +735,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range).must(date_range_mdate).must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-27")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-07")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+3
-4
@@ -232,12 +232,11 @@ class CommonDatautils {
|
||||
|
||||
public static def findRegionDistrictCount(def cat, def city, def region){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible", 0);
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("region", region);
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
return cursor.count();
|
||||
return DbUtils.seaweed.count(q);
|
||||
}
|
||||
|
||||
public static def findAllFiledDistrict(def cat, def city){
|
||||
@@ -4002,7 +4001,7 @@ class CommonDatautils {
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
|
||||
rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
|
||||
rets << ["id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
|
||||
// }
|
||||
|
||||
})
|
||||
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
package com.fangjia.script.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictFacilities2Batch extends BaseBatch {
|
||||
|
||||
public DBCollection onlineColl = MongoUtils.getCollectionByName("fangjia", "district_arround_poi", "mongo.db.fangjia.com")
|
||||
|
||||
public static void main(String[] args) {
|
||||
//小区周边数据更新
|
||||
this.runBatch(new DistrictFacilities2Batch(), args);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def cities = CommonDatautils.findAllCity()
|
||||
for (def city : cities) {
|
||||
def ret = CommonDatautils.getAllDistricts("district", city)
|
||||
def index = 0;
|
||||
for (def dbo : ret) {
|
||||
index++;
|
||||
println index + "/" + ret.size() + " " + dbo.city + " " + dbo.name
|
||||
|
||||
String region = dbo.get('region')
|
||||
String districtName = dbo.get('name')
|
||||
|
||||
if (!dbo.lat2 && !dbo.lng2) {
|
||||
continue;
|
||||
}
|
||||
def poms = findPoms(dbo.lat2 / 10000000000.0, dbo.lng2 / 10000000000.0);
|
||||
DBObject object = new BasicDBObject();
|
||||
object.put("hospital", unique(processType(poms.get('医院'), 1000, null, ['宠物', '动物', '口腔', '牙科', '美容', '齿科', '眼科', '号楼', '住院部', 'Pure', 'Tokushinkai', '假牙制作', '镶牙', '咪', '喵'])))
|
||||
object.put("shop", processType(poms.get('超市'), 1000, ['便民商店', '便利店'], ['杂货店', '烟酒', '爱鑫', '超市便利店']))
|
||||
object.put("university", processType(poms.get('学校'), 1000, ['大学'], ['食堂', '办公室', '德威', '早教', '夕阳红', '公司']))
|
||||
object.put("middle_school", processType(poms.get('学校'), 1000, ['中学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
|
||||
object.put("primary_school", processType(poms.get('学校'), 1000, ['小学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
|
||||
object.put("kindergarten", processType(poms.get('学校'), 1000, ['幼儿园', '托儿所'], ['食堂', '亲子', '办公室', '夕阳红', '公司']))
|
||||
object.put("food_shop", processType(poms.get('餐饮'), 500, ['餐饮服务']))
|
||||
def market = [];
|
||||
if (poms.get('商场')) {
|
||||
market = market + poms.get('商场');
|
||||
}
|
||||
if (poms.get('超市')) {
|
||||
market = market + poms.get('超市');
|
||||
}
|
||||
object.put("supermarket", processType(market, 1000, ['购物中心', "超市", "好又多"], ['公司', '村村寨寨', '伍缘折扣', '杂货店']))
|
||||
object.put("bank", processType(poms.get('银行'), 1000, ['银行'], ['自动提款机']))
|
||||
object.put("atm", processType(poms.get('ATM'), 1000, ['自动提款机', "atm", "ATM"]))
|
||||
updateToDistrictFacilitys(city, region, districtName, object)
|
||||
updateToDistrictFacilitysScore(city, region, districtName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findPoms(def lat, def lng) {
|
||||
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
|
||||
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put('lat', new BasicDBObject(QueryOperators.LTE, map.maxLat).append(QueryOperators.GTE, map.minLat));
|
||||
q.put('lng', new BasicDBObject(QueryOperators.LTE, map.maxLng).append(QueryOperators.GTE, map.minLng));
|
||||
def cursor = POI_COLL.find(q);
|
||||
def resMap = [:];
|
||||
for (def dbo : cursor) {
|
||||
def keyword = dbo.keyword;
|
||||
def details = resMap.get(keyword);
|
||||
if (!details) {
|
||||
details = [];
|
||||
resMap.put(keyword, details);
|
||||
}
|
||||
|
||||
def dMap = [:];
|
||||
dMap.name = dbo.name;
|
||||
dMap.lat = dbo.lat;
|
||||
dMap.lng = dbo.lng;
|
||||
dMap.distance = com.fangjia.script.util.GeoUtils.calculateDistance(lat, lng, dbo.lat, dbo.lng);
|
||||
details << dMap;
|
||||
}
|
||||
|
||||
resMap.each { k, v ->
|
||||
println "[${k}] " + v;
|
||||
}
|
||||
return resMap;
|
||||
}
|
||||
}
|
||||
@@ -566,6 +566,23 @@ class OpenApiUtils {
|
||||
return null;
|
||||
}
|
||||
|
||||
static def jsHouseCase(city, region, name, propertyTypeCode){
|
||||
def url = "http://localhost:8080/fangjia-open/house/case?city=${city}&district=${region}&buildingName=${name}&propertyTypeCode=${propertyTypeCode}"
|
||||
// println url
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def dto = []
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}else{
|
||||
println url
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def financialRating(city, address){
|
||||
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
|
||||
+61
-16
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.commStat
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
@@ -27,7 +28,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
|
||||
|
||||
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
|
||||
//for(def city : citys){
|
||||
@@ -308,21 +309,65 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
|
||||
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
|
||||
|
||||
//区域级统计
|
||||
def city = "合肥"
|
||||
def regions = CommonDatautils.getCityRegions(city)
|
||||
def rows = []
|
||||
for(def region : regions){
|
||||
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
|
||||
def alias_count = 0
|
||||
def addr_alias_count = 0
|
||||
for(def r : districts){
|
||||
if(r.alias && r.alias.size()>0){
|
||||
alias_count ++
|
||||
//def city = "合肥"
|
||||
//def regions = CommonDatautils.getCityRegions(city)
|
||||
//def rows = []
|
||||
//for(def region : regions){
|
||||
// def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
|
||||
// def alias_count = 0
|
||||
// def addr_alias_count = 0
|
||||
// for(def r : districts){
|
||||
// if(r.alias && r.alias.size()>0){
|
||||
// alias_count ++
|
||||
// }
|
||||
// if(r.addr_alias && r.addr_alias.size()>0){
|
||||
// addr_alias_count++
|
||||
// }
|
||||
// }
|
||||
// rows << [region, districts.size(), alias_count, addr_alias_count]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\区县楼盘量统计.xlsx",{d->
|
||||
if(d['C'] && !d['I']){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
if(!city){
|
||||
city = RegionData.formatCity(d['B'])
|
||||
}
|
||||
if(r.addr_alias && r.addr_alias.size()>0){
|
||||
addr_alias_count++
|
||||
if(city && d['D']){
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(region){
|
||||
def count = CommonDatautils.findRegionDistrictCount("district", city, region)
|
||||
println city +" "+ region+" "+count
|
||||
d['I'] = city
|
||||
d['J'] = region
|
||||
d['K'] = count
|
||||
}
|
||||
// def dists = CommonDatautils.findAllDistricts(d['C'], "district", ["city","region","name"])
|
||||
//d['B'] = dists.size()
|
||||
}
|
||||
}
|
||||
rows << [region, districts.size(), alias_count, addr_alias_count]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city",d['B'])
|
||||
// q.put("district","")
|
||||
// def dbo = mca_administrative_division_coll.findOne(q)
|
||||
// if(!dbo){
|
||||
// q.put("city", "省直辖县级行政区划")
|
||||
// q.put("district", d['B'])
|
||||
// dbo = mca_administrative_division_coll.findOne(q)
|
||||
// }
|
||||
// if(dbo){
|
||||
// d['A']=dbo.province
|
||||
// }
|
||||
})
|
||||
|
||||
//def citys = ["北京","上海","重庆","天津","石家庄","唐山","秦皇岛","邯郸","邢台","保定","张家口","承德","沧州","廊坊","衡水","太原","大同","阳泉","长治","晋城","朔州","晋中","运城","忻州","临汾","吕梁","呼和浩特","包头","乌海","赤峰","通辽","鄂尔多斯","呼伦贝尔","巴彦淖尔","乌兰察布","兴安盟","锡林郭勒盟","阿拉善盟","沈阳","大连","鞍山","抚顺","本溪","丹东","锦州","营口","阜新","辽阳","盘锦","铁岭","朝阳","葫芦岛","长春","吉林","四平","辽源","通化","白山","松原","白城","延边","哈尔滨","齐齐哈尔","鸡西","鹤岗","双鸭山","大庆","伊春","佳木斯","七台河","牡丹江","黑河","绥化","大兴安岭","南京","无锡","徐州","常州","苏州","南通","连云港","淮安","盐城","扬州","镇江","泰州","宿迁","杭州","宁波","温州","嘉兴","湖州","绍兴","金华","衢州","舟山","台州","丽水","合肥","芜湖","蚌埠","淮南","马鞍山","淮北","铜陵","安庆","黄山","滁州","阜阳","宿州","六安","亳州","池州","宣城","福州","厦门","莆田","三明","泉州","漳州","南平","龙岩","宁德","南昌","景德镇","萍乡","九江","新余","鹰潭","赣州","吉安","宜春","抚州","上饶","济南","青岛","淄博","枣庄","东营","烟台","潍坊","济宁","泰安","威海","日照","临沂","德州","聊城","滨州","菏泽","郑州","开封","洛阳","平顶山","安阳","鹤壁","新乡","焦作","濮阳","许昌","漯河","三门峡","南阳","商丘","信阳","周口","驻马店","武汉","黄石","十堰","宜昌","襄阳","鄂州","荆门","孝感","荆州","黄冈","咸宁","随州","恩施","长沙","株洲","湘潭","衡阳","邵阳","岳阳","常德","张家界","益阳","郴州","永州","怀化","娄底","湘西","广州","韶关","深圳","珠海","汕头","佛山","江门","湛江","茂名","肇庆","惠州","梅州","汕尾","河源","阳江","清远","东莞","中山","潮州","揭阳","云浮","南宁","柳州","桂林","梧州","北海","防城港","钦州","贵港","玉林","百色","贺州","河池","来宾","崇左","海口","三亚","三沙","儋州","成都","自贡","攀枝花","泸州","德阳","绵阳","广元","遂宁","内江","乐山","南充","眉山","宜宾","广安","达州","雅安","巴中","资阳","阿坝州","甘孜州","凉山州","贵阳","六盘水","遵义","安顺","毕节","铜仁","黔西南","黔东南","黔南","昆明","曲靖","玉溪","保山","昭通","丽江","普洱","临沧","楚雄","红河","文山州","西双版纳州","大理","德宏州","怒江州","迪庆州","拉萨","日喀则","昌都","林芝","山南","那曲","阿里","西安","铜川","宝鸡","咸阳","渭南","延安","汉中","榆林","安康","商洛","兰州","嘉峪关","金昌","白银","天水","武威","张掖","平凉","酒泉","庆阳","定西","陇南","临夏","甘南","西宁","海东","海北州","黄南州","海南州","果洛州","玉树州","海西州","银川","石嘴山","吴忠","固原","中卫","乌鲁木齐","克拉玛依","吐鲁番","哈密","昌吉","博尔塔拉","巴音郭楞","阿克苏","克孜勒苏","喀什","和田","伊犁","塔城","阿勒泰","济源","仙桃","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","石河子","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉","昆玉"]
|
||||
//
|
||||
//def fjwCitys = CommonDatautils.findAllCity()
|
||||
//for(def fjw_city : fjwCitys){
|
||||
// if(!citys.contains(fjw_city)){
|
||||
// println fjw_city
|
||||
// }
|
||||
//}
|
||||
+152
-3
@@ -5,6 +5,7 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@@ -22,12 +23,16 @@ class ApiTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new ApiTest()
|
||||
// task.rankRunTest()
|
||||
// task.similarRunTest()
|
||||
task.loupanLevelTest()
|
||||
// task.rankRunTest()//小区排名接口测试
|
||||
// task.similarRunTest()//相似房源接口测试
|
||||
// task.loupanLevelTest()//楼盘评级接口测试
|
||||
|
||||
// task.jsAbcHouseCastTest()//江苏农总行房源接口测试
|
||||
task.arroundFacilities()//周边poi
|
||||
}
|
||||
|
||||
|
||||
|
||||
def loupanLevelTest(){
|
||||
ExcelUtils.modify("D:\\数据\\卓信\\上海 - 副本.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
@@ -125,6 +130,150 @@ class ApiTest {
|
||||
return dbo
|
||||
}
|
||||
|
||||
def jsAbcHouseCastTest(){
|
||||
def citys = ["无锡"]
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","estate_type2"])
|
||||
def rets = getHouseCases(districts)
|
||||
ExcelUtils.write("D:\\数据\\江苏\\房源样本.xlsx", rets, city)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def getHouseCases(districts){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
rets << ["城市","区域","名称", "物业类型", "最小案例","最大案例", "案例"]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def propertyTypeCode = null
|
||||
if(d.estate_type2=="普通住宅"){
|
||||
propertyTypeCode = "01"
|
||||
}else if(d.estate_type2=="别墅"){
|
||||
propertyTypeCode = "02"
|
||||
}else if(d.estate_type2=="商住"){
|
||||
propertyTypeCode = "03"
|
||||
}
|
||||
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCase(d.city, d.region, d.name, propertyTypeCode)
|
||||
if(results){
|
||||
def row = []
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
row << d.estate_type2
|
||||
row << results.minUnitPrice
|
||||
row << results.maxUnitPrice
|
||||
|
||||
def datas = results.datas
|
||||
if(datas){
|
||||
def str = ""
|
||||
for(def r : datas){
|
||||
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
|
||||
}
|
||||
row << str
|
||||
}
|
||||
|
||||
rets << row
|
||||
}
|
||||
println totalCount.get()+" "+d.city +" "+d.region+" "+d.name
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def arroundFacilities(){
|
||||
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city","region","name","estate_type2"])
|
||||
// districts = districts[0..1000]
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
rets << ["城市","区域","名称", "重点学校","学校", "公交","地铁","超市","医院","银行"]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def row = []
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
def facilityObj = OpenApiClient.facilitys(d.city, d.region, d.name)
|
||||
if(facilityObj && facilityObj.result){
|
||||
def result = facilityObj.result
|
||||
if(result.primarySchool){
|
||||
row <<result.primarySchool.join(",").replace('"',"")
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.school){
|
||||
row << result.school.join(",").replace('"',"")
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.bus){
|
||||
row << result.bus.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
|
||||
if(result.metro){
|
||||
row << result.metro.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
|
||||
if(result.supermarket){
|
||||
row << result.supermarket.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.hospital){
|
||||
row << result.hospital.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.bank){
|
||||
row << result.bank.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
println totalCount.get() + " "+d.region +" "+d.name
|
||||
rets << row
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\江苏\\南京周边配套.xlsx", rets, "南京")
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+40
-1
@@ -21,7 +21,8 @@ class MsbPocTest {
|
||||
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
|
||||
// test("D:\\数据\\美术宝\\test2.xlsx")
|
||||
|
||||
cityPrice()
|
||||
// cityPrice()
|
||||
mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
|
||||
}
|
||||
|
||||
def cityPrice(){
|
||||
@@ -130,6 +131,44 @@ class MsbPocTest {
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def mergeMatchPrice(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['L']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def name = row['F']
|
||||
def address = row['G']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, "property", false)
|
||||
if(jsonObject && !jsonObject.containsKey("result")){
|
||||
jsonObject = FangjiaClientUtils.newMatchPrice(city, address, "property", false)
|
||||
}
|
||||
if(jsonObject && !jsonObject.containsKey("result")){
|
||||
jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, null, true)
|
||||
}
|
||||
|
||||
if(jsonObject && !jsonObject.containsKey("result")){
|
||||
jsonObject = FangjiaClientUtils.newMatchPrice(city, address, null, true)
|
||||
}
|
||||
if(jsonObject && jsonObject.containsKey("result")) {
|
||||
def resultMap = jsonObject.get("result")
|
||||
row['L'] = resultMap.get("city")
|
||||
row['M'] = resultMap.get("district")
|
||||
row['N'] = resultMap.get("name")
|
||||
def type = resultMap.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['O'] = type
|
||||
}
|
||||
if(resultMap.get("avgPrice")){
|
||||
row['P'] = resultMap.get("avgPrice")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
|
||||
+286
@@ -0,0 +1,286 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.*
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL)
|
||||
|
||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||
|
||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
public static def priceMap = null
|
||||
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
def keys = new CopyOnWriteArraySet();
|
||||
def fullCityMap = null
|
||||
def mcaDistMap = null
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SuggestionDataSynch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
run()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run() {
|
||||
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
fullCityMap = queryFullNameCity()
|
||||
mcaDistMap = queryDistNameCity()
|
||||
|
||||
for (def c : citys) {
|
||||
index++
|
||||
if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
||||
updating(c, "district")
|
||||
updating(c, "office")
|
||||
updating(c, "shop")
|
||||
println index + " " + c + " 完成"
|
||||
|
||||
}
|
||||
|
||||
// poiData(c)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
def key = dist.city+"#"+dist.region+"#"+dist.name
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
|
||||
def map = createData(dist)
|
||||
//往100 上同步
|
||||
updateData(map)
|
||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def createData(dist){
|
||||
def map = [:]
|
||||
map.id = dist._id.toString()
|
||||
def province = provinceMap.get(dist.city)
|
||||
if(province){
|
||||
map.province = ProvinceData.getFullProvince(province)
|
||||
}
|
||||
|
||||
def data = fullCityMap.get(dist.city)
|
||||
map.city = data.city
|
||||
|
||||
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
|
||||
if(distData){
|
||||
map.region = distData.district
|
||||
}else{
|
||||
map.region = dist.region
|
||||
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
|
||||
}
|
||||
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
if(dist.alias && dist.alias.size()>0){
|
||||
map.alias = dist.alias
|
||||
}
|
||||
if(dist.addr_alias && dist.addr_alias.size()>0){
|
||||
map.addr_alias = dist.addr_alias
|
||||
}
|
||||
map.type = dist.cat
|
||||
if(dist.lng){
|
||||
def loc = [:]
|
||||
loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0)
|
||||
loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0)
|
||||
map.location = loc
|
||||
}
|
||||
map.source = "FJW"
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findAllDistrictSellPrice(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("date", lastMonthDate)
|
||||
def priceMap = [:]
|
||||
DISTRICT_SELL_STAT.find(q).each{
|
||||
priceMap.put(it.commid, it)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
|
||||
def updateData(map){
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(k!="id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
SUGGESTION_CONN.update(new BasicDBObject('_id', map.id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
|
||||
|
||||
def poiData(city){
|
||||
def data = fullCityMap.get(city)
|
||||
if(data){
|
||||
def results = CommonDatautils.findAmapDetailBySql(data.city)
|
||||
createPoi(results, data)
|
||||
println city +" 完成!"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def createPoi(results, data){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
for(def i=0;i<results.size();i++){
|
||||
final def dist = results[i]
|
||||
def key = dist.city+"#"+dist.district+"#"+dist.name
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
|
||||
def map = [:]
|
||||
map.id = dist.id
|
||||
map.province = dist.province
|
||||
map.city = dist.city
|
||||
map.region = dist.district
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
map.type = "poi"
|
||||
map.cat = dist.cat
|
||||
map.location = dist.location
|
||||
map.source = "FJW"
|
||||
//往100 上同步
|
||||
updateData(map)
|
||||
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def fjwcity = null
|
||||
if(d.level == "城市"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(!fjwcity){
|
||||
if(d.city == "市辖区"){
|
||||
fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
d.city = d.province
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(d.city == "省直辖县级行政区划" && d.district){
|
||||
fjwcity = RegionData.formatCity(d.district)
|
||||
if(fjwcity){
|
||||
d.city = d.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(fjwcity){
|
||||
retMap.put(fjwcity, d)
|
||||
}else{
|
||||
println d.toString() +" 格式化不成功!"
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("level","区域")
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.city == "省直辖县级行政区划"){
|
||||
continue
|
||||
}else if(d.city == "市辖区"){
|
||||
def fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}else{
|
||||
def fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
}
|
||||
+1
-2
@@ -765,9 +765,8 @@ public class OpenApiClient {
|
||||
params.put("region", region)
|
||||
params.put("name", name)
|
||||
try {
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ public class ProvinceData {
|
||||
put("辽宁", ["辽宁省", "辽"]);
|
||||
put("黑龙江", ["黑龙江省", "黑"]);
|
||||
put("湖南", ["湖南省", "湘"]);
|
||||
put("湖北", ["湖北省", "鄂"]);
|
||||
put("安徽", ["安徽省", "皖"]);
|
||||
put("山东", ["山东省", "鲁"]);
|
||||
put("新疆", ["新疆维吾尔自治区", "新"]);
|
||||
@@ -62,6 +63,38 @@ public class ProvinceData {
|
||||
public static Map<String, String> CITY_PROVINCE_MAP = new HashMap<String, String>();
|
||||
static DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com", 27017);
|
||||
|
||||
|
||||
static final Map<String, String> PROVINCE_MAP = new HashMap<String, String>() {{
|
||||
put("河北", "河北省");
|
||||
put("河南", "河南省");
|
||||
put("云南", "云南省");
|
||||
put("辽宁", "辽宁省");
|
||||
put("黑龙江", "黑龙江省");
|
||||
put("湖南", "湖南省");
|
||||
put("湖北", "湖北省");
|
||||
put("安徽", "安徽省");
|
||||
put("山东", "山东省");
|
||||
put("新疆", "新疆维吾尔自治区");
|
||||
put("江苏", "江苏省");
|
||||
put("浙江", "浙江省");
|
||||
put("江西", "江西省");
|
||||
put("广西", "广西壮族自治区");
|
||||
put("甘肃", "甘肃省");
|
||||
put("山西", "山西省");
|
||||
put("内蒙古", "内蒙古自治区");
|
||||
put("陕西", "陕西省");
|
||||
put("吉林", "吉林省");
|
||||
put("福建", "福建省");
|
||||
put("贵州", "贵州省");
|
||||
put("广东", "广东省");
|
||||
put("青海", "青海省");
|
||||
put("西藏", "西藏自治区");
|
||||
put("四川", "四川省");
|
||||
put("宁夏", "宁夏回族自治区");
|
||||
put("海南", "海南省");
|
||||
put("台湾", "台湾省");
|
||||
}};
|
||||
|
||||
static {
|
||||
logger.info("开始加载省份信息");
|
||||
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("province", 1).append("display", 1)).batchSize(300);
|
||||
@@ -109,6 +142,16 @@ public class ProvinceData {
|
||||
return CITY_PROVINCE_MAP.get(city);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 根据省的简称获取省的全称
|
||||
* @param province
|
||||
* @return
|
||||
*/
|
||||
public static String getFullProvince(String province){
|
||||
return PROVINCE_MAP.get(province);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取省份下的所有城市
|
||||
* @param province
|
||||
|
||||
+2
-1
@@ -11,6 +11,7 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def fileStartDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230301", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
@@ -38,7 +39,7 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
ExcelUtils.write("D:\\亲邻数据${fileStartDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
//def list = []
|
||||
|
||||
+3
-3
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230901", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20231001", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20231001", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20231101", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -39,6 +39,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202310.xlsx", data, "楼盘评情")
|
||||
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202311.xlsx", data, "楼盘评情")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user