Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -2,36 +2,35 @@ package com.fangjia.script
|
||||
|
||||
public enum Watcher {
|
||||
//ÀëְͬÊÂ
|
||||
LiuPeiPei('13621626685', 'duxiaoyu@fangjia.com'),
|
||||
YanZhou('13621626685','duxiaoyu@fangjia.com'),
|
||||
HuangXiaoHu('13621626685', 'duxiaoyu@fangjia.com'),
|
||||
SongYingXia('13621626685', 'duxiaoyu@fangjia.com'),
|
||||
YuKai('13621626685', 'duxiaoyu@fangjia.com'),
|
||||
RuanYaDong('13621626685', 'duxiaoyu@fangjia.com'),
|
||||
Reda('13621626685', 'duxiaoyu@fangjia.com'),
|
||||
Tomzhao('13621626685','duxiaoyu@fangjia.com'),
|
||||
YuHai('13621626685','duxiaoyu@fangjia.com'),
|
||||
Shibb('13621626685','duxiaoyu@fangjia.com'),
|
||||
HeLvYuan('13621626685','duxiaoyu@fangjia.com'),
|
||||
WangYuHui('13621626685','duxiaoyu@fangjia.com'),
|
||||
LinXiaoShuai('15801810393','linxiaoshuai@fangjia.com'),
|
||||
FangRui('17717680589','fangrui@fangjia.com'),
|
||||
//DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'),
|
||||
LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'),
|
||||
YanZhou('13621626685','wangwenyan@fangjia.com'),
|
||||
HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'),
|
||||
SongYingXia('13621626685', 'wangwenyan@fangjia.com'),
|
||||
YuKai('13621626685', 'wangwenyan@fangjia.com'),
|
||||
RuanYaDong('13621626685', 'wangwenyan@fangjia.com'),
|
||||
Reda('13621626685', 'wangwenyan@fangjia.com'),
|
||||
Tomzhao('13621626685','wangwenyan@fangjia.com'),
|
||||
YuHai('13621626685','wangwenyan@fangjia.com'),
|
||||
Shibb('13621626685','wangwenyan@fangjia.com'),
|
||||
HeLvYuan('13621626685','wangwenyan@fangjia.com'),
|
||||
WangYuHui('13621626685','wangwenyan@fangjia.com'),
|
||||
LinXiaoShuai('15801810393','wangwenyan@fangjia.com'),
|
||||
FangRui('17717680589','wangwenyan@fangjia.com'),
|
||||
DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
|
||||
|
||||
//ÔÚְͬÊÂ
|
||||
DuXiaoYu('13916076378', 'wangwenyan@fangjia.com'),
|
||||
LiXue('13311649646','lixue@fangjia.com'),
|
||||
LiXue('13311649646','wangwenyan@fangjia.com'),
|
||||
WangWenYan('13916076378','wangwenyan@fangjia.com'),
|
||||
GaoJuan('15021384995','gaojuan@fangjia.com'),
|
||||
Hubingliang('15179856208','hubingliang@126.com'),
|
||||
ZhouTongYun('13585834921','zhoutongyun@fangjia.com'),
|
||||
YinJiHuan('18217642009','yinjihuan@fangjia.com'),
|
||||
Xuhe('13862416422','xuhe@fangjia.com'),
|
||||
ZongZhiMin('18260046486','zongzhimin@fangjia.com'),
|
||||
RaoYanHui('18516539752','raoyanhui@fangjia.com'),
|
||||
BaYi('15300424133','bayi@fangjia.com'),
|
||||
WeiLei('18101751850','weilei@fangjia.com'),
|
||||
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
|
||||
Hubingliang('15179856208','wangwenyan@fangjia.com'),
|
||||
ZhouTongYun('13585834921','wangwenyan@fangjia.com'),
|
||||
YinJiHuan('18217642009','wangwenyan@fangjia.com'),
|
||||
Xuhe('13862416422','wangwenyan@fangjia.com'),
|
||||
ZongZhiMin('18260046486','wangwenyan@fangjia.com'),
|
||||
RaoYanHui('18516539752','wangwenyan@fangjia.com'),
|
||||
BaYi('15300424133','wangwenyan@fangjia.com'),
|
||||
WeiLei('18101751850','wangwenyan@fangjia.com'),
|
||||
JiangYaHui('13684913780','wangwenyan@fangjia.com'),
|
||||
ShenJian('13764369683', 'shenjian@fangjia.com')
|
||||
private final String mobile;
|
||||
private final String email;
|
||||
|
||||
+7
-7
@@ -124,13 +124,13 @@ public class ManualOperationDataStat extends BaseBatch{
|
||||
println(content)
|
||||
sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
// sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.script.ana
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -40,9 +42,46 @@ public class Test {
|
||||
// //test.run("district", "上海");
|
||||
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
|
||||
// println new Date(new ObjectId('5469bb4cda29ee73010fa577').getTime())
|
||||
test.addTestData();
|
||||
// test.addTestData();
|
||||
test.findRepeatGuidingPrice()
|
||||
}
|
||||
|
||||
def findRepeatGuidingPrice(){
|
||||
def q = new BasicDBObject();
|
||||
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-03-01", "yyyy-MM-dd")))
|
||||
def dbcursor = GUIDING_PRICE_COLL.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.region+"##"+d.name
|
||||
def rets = map.get(key)
|
||||
if(!rets){
|
||||
rets = []
|
||||
map.put(key, rets)
|
||||
}
|
||||
rets << d
|
||||
}
|
||||
|
||||
def datas = []
|
||||
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
|
||||
map.each{key,list->
|
||||
if(list.size()>1){
|
||||
def userNames = []
|
||||
for(def ret : list){
|
||||
userNames << ret.guiding_person
|
||||
}
|
||||
if(userNames.contains("付贵生")){
|
||||
for(def r : list){
|
||||
if(r.guiding_person!="付贵生"){
|
||||
GUIDING_PRICE_COLL.remove(new BasicDBObject("_id", r._id))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void addTestData(){
|
||||
|
||||
+5
-6
@@ -28,7 +28,8 @@ import org.bson.types.ObjectId
|
||||
class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
|
||||
|
||||
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
|
||||
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
|
||||
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017)
|
||||
@@ -215,7 +216,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("city", city)
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
query.put("status", 0)
|
||||
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
@@ -227,11 +228,9 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
int tryTimes = 5
|
||||
List<BasicDBObject> districts = new ArrayList<BasicDBObject>()
|
||||
for(int i=0; i < tryTimes; i++) {
|
||||
DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
|
||||
def cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
|
||||
int totalCount = cur.count()
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
while(iter.hasNext()) {
|
||||
BasicDBObject dbo = (BasicDBObject)iter.next()
|
||||
for(def dbo : cur) {
|
||||
districts.add(dbo)
|
||||
}
|
||||
if(districts.size() == totalCount) {
|
||||
|
||||
+11
-5
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -58,10 +58,16 @@ for(def d : datas){
|
||||
}catch(Exception e){
|
||||
println "room error"
|
||||
}
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
if(arrs[1].toString()){
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
}
|
||||
if(arrs.size()==4 && arrs[3].toString()){
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
}
|
||||
}else{
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
if(roomStr.toString()){
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -74,7 +80,7 @@ for(def d : datas){
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* 添加人工指导价,并重算价格
|
||||
* 小付的人工指导价不能覆盖
|
||||
*/
|
||||
class BatchAddPersonGuiPrice {
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new BatchAddPersonGuiPrice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx"
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
for(def d : datas){
|
||||
def row = findGuidingPrice(d['B'], d['C'], d['D'])
|
||||
if(!row){
|
||||
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2)
|
||||
//发重算均价命令
|
||||
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findGuidingPrice(city, region, name) {
|
||||
def q = new BasicDBObject();
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("guiding_person" , "付贵生")
|
||||
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd")))
|
||||
def dbo = GUIDING_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def save(city, region, name, price, estate_type){
|
||||
DBObject obj = new BasicDBObject();
|
||||
obj.put("city", city);
|
||||
obj.put("region", region);
|
||||
obj.put("name", name);
|
||||
obj.put("guiding_date", new Date()-7);
|
||||
obj.put("guiding_value", price.toString().toDouble().toInteger());
|
||||
obj.put("category", "district");
|
||||
obj.put("guiding_type", 0);
|
||||
obj.put("c_date", new Date());
|
||||
obj.put("m_date", new Date());
|
||||
obj.put("estate_type" , estate_type);
|
||||
obj.put("house_type" , 0)
|
||||
obj.put("guiding_person" , "陈杰")
|
||||
GUIDING_PRICE_COLL.save(obj);
|
||||
}
|
||||
|
||||
|
||||
def addPrice(city, region, name, estateType2,startDate, endDate){
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+14
-1
@@ -90,11 +90,16 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
// delPgData()
|
||||
// saveToPgSql2(datas)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
@@ -311,7 +316,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
def index = 0
|
||||
sqlInstance.withBatch(10000, sql){ ps->
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : newRows) {
|
||||
index++
|
||||
try{
|
||||
@@ -339,4 +344,12 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
return rows;
|
||||
}
|
||||
|
||||
def updateVersion(def time){
|
||||
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
|
||||
for(def tableName : tables){
|
||||
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+80
-17
@@ -39,7 +39,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
@@ -60,20 +60,14 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
// }
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
@@ -84,7 +78,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def dealIds = findHistoryDealIds(city)
|
||||
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
println "历史id加载量:"+dealIds.size()
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
@@ -115,11 +109,29 @@ class BigdataDealBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
// delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
// saveToPgSql2(datas)
|
||||
}
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split("\\|@\\|")
|
||||
rows << row
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
dealIds = Collections.synchronizedList(dealIds);
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
@@ -147,12 +159,14 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
dealIds << trade._id.toString()
|
||||
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
}
|
||||
}else{
|
||||
def ret = converData(trade, cityCodeDatas, distCodeMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
dealIds << trade._id.toString()
|
||||
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
@@ -167,6 +181,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -217,7 +232,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def price = ccbPriceMap.get(obj._id.toString())
|
||||
if(price){
|
||||
def mod = java.lang.Math.abs((trade.price-price)*100/price)
|
||||
if(mod<=10){
|
||||
if(mod<=20){
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
@@ -515,6 +530,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
|
||||
println city +" 小区量:"+dists.size()
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
@@ -635,7 +651,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def findHistoryDealIds(def city){
|
||||
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}' "
|
||||
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'"
|
||||
def ids = []
|
||||
try{
|
||||
pgsqlInstance.eachRow(sql,{
|
||||
@@ -723,6 +739,14 @@ class BigdataDealBatch extends BaseBatch{
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def delPgData(){
|
||||
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
pgsqlInstance.execute("delete from base.t0831_bigdata_deal_gj_h where p9_start_date='${month}'")
|
||||
println "base.t0831_bigdata_deal_gj_h 数据已删除"
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
@@ -753,4 +777,43 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql2(rows){
|
||||
def newRows = []
|
||||
for(def r : rows) {
|
||||
def newR = []
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
// r[i] = null
|
||||
newR << null
|
||||
}else{
|
||||
newR << r[i]
|
||||
}
|
||||
}
|
||||
newR << null
|
||||
newR << "2023-09-28"
|
||||
newR << null
|
||||
newR << '2999-12-31'
|
||||
newR << null
|
||||
newR << null
|
||||
newR << null
|
||||
|
||||
newRows << newR
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : newRows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
|
||||
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
|
||||
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
|
||||
+6
-2
@@ -85,6 +85,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
|
||||
def startDate = lastMonth+"18"
|
||||
def endDate = curMonth+"19"
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
@@ -196,6 +197,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
@@ -311,7 +313,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.room_structure_code==null?"":m.room_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
@@ -670,6 +672,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
@@ -731,7 +734,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range).must(date_range_mdate)
|
||||
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
@@ -882,6 +885,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
|
||||
+192
@@ -0,0 +1,192 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
/***
|
||||
* 同步上个月城市,区域,小区价格到base库中
|
||||
*/
|
||||
class BigdataSynchPrice extends BaseBatch{
|
||||
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataSynchPrice(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
synchCityPrice(citys)
|
||||
synchDistrictPrice(citys)
|
||||
synchCommPirce(citys)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def synchCityPrice(citys){
|
||||
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_city_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_trdpt_avprc_id
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujiabds_clc_yrmo_val
|
||||
data << r.gujia_bds_clc_avprc
|
||||
data << r.gujia_bds_urbn_avprc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
|
||||
data << null
|
||||
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
|
||||
data << null
|
||||
data << '2999-12-31'
|
||||
data << null
|
||||
data << null
|
||||
data << null
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
println "同步城市均价 start: "+new Date()
|
||||
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(500, sql){ ps->
|
||||
for(def r : allRows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def synchDistrictPrice(citys){
|
||||
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'"; //
|
||||
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_trdpt_avprc_id
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gxda_admndiv_nm
|
||||
data << r.gujiabds_clc_yrmo_val
|
||||
data << r.gujia_bds_clc_avprc
|
||||
data << r.gujia_bds_urbn_avprc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
|
||||
data << null
|
||||
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
|
||||
data << null
|
||||
data << '2999-12-31'
|
||||
data << null
|
||||
data << null
|
||||
data << null
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
println "同步区域均价 start: "+new Date()
|
||||
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(1000, sql){ ps->
|
||||
for(def r : allRows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def synchCommPirce(citys){
|
||||
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_bss_prc_id
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.gujia_bds_id
|
||||
data << r.comm_nm
|
||||
data << r.gujiabds_clc_yrmo_val
|
||||
data << r.gujia_bds_clc_avprc
|
||||
data << r.gujia_comm_basic_prc
|
||||
data << r.gujia_bds_clmtd
|
||||
data << r.gujiabds_fwd_samp_rto
|
||||
data << r.gujia_bds_mns_grd
|
||||
data << r.gujia_bds_cfdrg
|
||||
data << r.gujiabdsarndcmprccrlt
|
||||
data << r.gujia_bds_cfdc_dgr
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
|
||||
data << null
|
||||
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
|
||||
data << null
|
||||
data << '2999-12-31'
|
||||
data << null
|
||||
data << null
|
||||
data << null
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
println "同步小区价格 start: "+new Date()
|
||||
def sql = "insert into base.t0831_community_base_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : allRows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
+29
-6
@@ -30,6 +30,9 @@ class CcbCommonUtils {
|
||||
|
||||
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
@@ -37,7 +40,7 @@ class CcbCommonUtils {
|
||||
|
||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42","xflapp.com":"43"]
|
||||
|
||||
public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"]
|
||||
|
||||
@@ -225,7 +228,7 @@ class CcbCommonUtils {
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
@@ -432,7 +435,7 @@ class CcbCommonUtils {
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","金地玺湾")
|
||||
// q.put("name","汤堡小区")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
@@ -443,7 +446,7 @@ class CcbCommonUtils {
|
||||
}
|
||||
}
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "[${city}] 原始:"+dbsour.count()
|
||||
def rets = []
|
||||
|
||||
@@ -467,7 +470,7 @@ class CcbCommonUtils {
|
||||
public static def findAllDistrictByCityAndRegion(def city, def region, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("region", region)
|
||||
@@ -482,7 +485,7 @@ class CcbCommonUtils {
|
||||
}
|
||||
}
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "原始:"+dbsour.count()
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
@@ -679,4 +682,24 @@ class CcbCommonUtils {
|
||||
return ids;
|
||||
}
|
||||
|
||||
|
||||
static def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+50
-54
@@ -29,7 +29,7 @@ class DealPriceDataProcess {
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
task.deleteThirdPartyData()//删除第三方
|
||||
// task.delDealByPe()
|
||||
// task.delDealByPe()//删除成交
|
||||
// task.delDealData()
|
||||
|
||||
// task.exportThirdParty()
|
||||
@@ -37,49 +37,49 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
def yearMonth = 202201
|
||||
def thirdMonth = 202112
|
||||
def yearMonth = 202401
|
||||
|
||||
// def rets = findAllDealByPe(yearMonth, 70)
|
||||
// for(def r : rets){
|
||||
// deleteDealPrice(r)
|
||||
def rets = findAllDealByPe(yearMonth, 18)
|
||||
|
||||
for(def r : rets){
|
||||
deleteDealPrice(r)
|
||||
}
|
||||
// def list = findAllThirdByPe(yearMonth, 70)
|
||||
// for(def r : list){
|
||||
// deletThirdPrice(r, thirdMonth)
|
||||
// }
|
||||
// def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
|
||||
// for(def d : datas){
|
||||
//// if(d['G']){
|
||||
//// def rets = findDealByPe(d['A'], yearMonth, d['G'])
|
||||
//// for(def r : rets){
|
||||
//// deleteDealPrice(r)
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// if(d['H']){
|
||||
//// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
|
||||
//// for(def m : rets){
|
||||
//// def precent = d['H']/100.0
|
||||
//// def newPrice = null
|
||||
//// if(m.unit_price > m.deal_price){
|
||||
//// newPrice = m.deal_price*precent + m.deal_price
|
||||
//// }else if(m.unit_price < m.deal_price){
|
||||
//// newPrice = m.deal_price - (m.deal_price*precent)
|
||||
//// }
|
||||
//// if(newPrice){
|
||||
//// updateDealPrice(m, newPrice.toInteger())
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// if(d['M']){
|
||||
//// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
|
||||
//// for(def r : rets){
|
||||
//// deletThirdPrice(r, thirdMonth)
|
||||
//// }
|
||||
//// }
|
||||
// }
|
||||
def list = findAllThirdByPe(yearMonth, 70)
|
||||
for(def r : list){
|
||||
deletThirdPrice(r, thirdMonth)
|
||||
}
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
|
||||
for(def d : datas){
|
||||
// if(d['G']){
|
||||
// def rets = findDealByPe(d['A'], yearMonth, d['G'])
|
||||
// for(def r : rets){
|
||||
// deleteDealPrice(r)
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(d['H']){
|
||||
// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
|
||||
// for(def m : rets){
|
||||
// def precent = d['H']/100.0
|
||||
// def newPrice = null
|
||||
// if(m.unit_price > m.deal_price){
|
||||
// newPrice = m.deal_price*precent + m.deal_price
|
||||
// }else if(m.unit_price < m.deal_price){
|
||||
// newPrice = m.deal_price - (m.deal_price*precent)
|
||||
// }
|
||||
// if(newPrice){
|
||||
// updateDealPrice(m, newPrice.toInteger())
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(d['M']){
|
||||
// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
|
||||
// for(def r : rets){
|
||||
// deletThirdPrice(r, thirdMonth)
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -151,9 +151,10 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def deleteDealPrice(r){
|
||||
def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
|
||||
"and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
// def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
|
||||
// "and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
|
||||
def sql = "delete from base.t0831_bigdata_deal_gj_h where gujia_comm_id='${r.commid}' and txn_dt='${r.dealdate}' and guiabconstructionarea=${r.buildarea} and gujia_bds_trdtckt_prc=${r.deal_price}"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}
|
||||
|
||||
def deletThirdPrice(r, thirdMonth){
|
||||
@@ -177,19 +178,14 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def findAllDealByPe(yearMonth, peVal){
|
||||
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='成交' and pe>${peVal}";
|
||||
def sql = "select gujia_comm_id as commid,gujiabds_valt_unitprc as unit_price,gujiabds_to_mkprc_mny as target_avg_price,gujia_area as buildarea,to_char(mdl_dt,'yyyy-mm-dd') as dealdate from t_app_p9pha_plda.avm_evaluation_item_gj_h where urbn_nm in ('昆明','南京','苏州','重庆') and gujiabds_clc_yrmo_val=${yearMonth} and gujia_data_tp='成交' and gujia_mns_rto>${peVal} ";
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.cityname = r.cityname
|
||||
map.cityid=r.cityid
|
||||
map.distid = r.distid
|
||||
map.commid = r.commid
|
||||
map.commname = r.commname
|
||||
map.deal_price = r.target_avg_price
|
||||
map.unit_price = r.unit_price
|
||||
map.buildarea = r.buildarea
|
||||
map.ppe = r.pe
|
||||
map.dealdate = r.dealdate
|
||||
rets << map
|
||||
})
|
||||
@@ -392,12 +388,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202306
|
||||
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202307)
|
||||
def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
|
||||
+2
-10
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -84,8 +84,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
saveToPgSql(rows)
|
||||
//
|
||||
//更新静态数据表
|
||||
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -244,14 +243,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
def updateVersion(def time){
|
||||
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
|
||||
for(def tableName : tables){
|
||||
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+83
-52
@@ -2,6 +2,7 @@ package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.RandomUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
@@ -1287,10 +1288,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
|
||||
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
@@ -1299,37 +1300,42 @@ def fileName = "D:\\
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
//// if(!d['F']){
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//////
|
||||
//// if(dist){
|
||||
////// d['H'] = dist.region
|
||||
////// d['G'] = dist.name
|
||||
//// d['F'] = dist.status
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
if(!d['I']){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if(dist){
|
||||
d['F'] = dist.region
|
||||
// d['H'] = dist.name
|
||||
d['G'] = dist.status
|
||||
// if(d['A']=="天门" || d['A']=="迪庆"){
|
||||
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
|
||||
// d['K'] = val.toInteger()
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
|
||||
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
|
||||
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
// if(count && count==1){
|
||||
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
|
||||
// d['E'] = dist.city
|
||||
//// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
//// if(count && count==1){
|
||||
//// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
|
||||
//// d['E'] = dist.city
|
||||
//// }
|
||||
////
|
||||
// if(d['F']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['F'])
|
||||
// if(dist){
|
||||
// d['G'] = dist.status
|
||||
// d['H'] = dist.name_cat
|
||||
// d['I'] = dist.estate_type2
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['J'] = dist.status
|
||||
//// }
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
//// d['F'] = newRegion
|
||||
////// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
////// d['F'] = newRegion
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify(fileName, {d->
|
||||
@@ -1373,30 +1379,30 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
|
||||
return count
|
||||
}
|
||||
|
||||
citys = CcbCommonUtils.citys_all
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
|
||||
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
|
||||
def rows = []
|
||||
def headers = []
|
||||
headers << "城市"
|
||||
headers.addAll(dates)
|
||||
rows << headers
|
||||
for(def c : citys){
|
||||
def row = []
|
||||
row << c
|
||||
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
|
||||
// for(def d : dates){
|
||||
def startDate = "2022-07-01"
|
||||
def count = findPgDealPrice(c, pgsqlInstance, startDate)
|
||||
row << count*100/disCount
|
||||
|
||||
// def map = findSampleRate(pgsqlInstance, c, d)
|
||||
// row << map.third
|
||||
// }
|
||||
rows << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
|
||||
//citys = CcbCommonUtils.citys_all
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
|
||||
//def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
|
||||
//def rows = []
|
||||
//def headers = []
|
||||
//headers << "城市"
|
||||
//headers.addAll(dates)
|
||||
//rows << headers
|
||||
//for(def c : citys){
|
||||
// def row = []
|
||||
// row << c
|
||||
// def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
|
||||
//// for(def d : dates){
|
||||
// def startDate = "2022-07-01"
|
||||
// def count = findPgDealPrice(c, pgsqlInstance, startDate)
|
||||
// row << count*100/disCount
|
||||
//
|
||||
//// def map = findSampleRate(pgsqlInstance, c, d)
|
||||
//// row << map.third
|
||||
//// }
|
||||
// rows << row
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
|
||||
|
||||
|
||||
|
||||
@@ -1414,6 +1420,31 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){
|
||||
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\房源\\仙桃.xlsx", {d->
|
||||
// def dist = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
|
||||
// def flag = null;
|
||||
// if(!dist){
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'],d['B'],d['C'],"district")
|
||||
// flag = "映射别名匹配"
|
||||
// if(!dist) {
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'],d['B'],d['C'], "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// flag = "别名匹配"
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(dist){
|
||||
// d['D'] = dist.status
|
||||
// d['E'] = dist.estate_type2
|
||||
// d['F'] = flag
|
||||
// d['G'] = dist.name
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -54,7 +54,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
// ps.addBatch(id)
|
||||
// }
|
||||
// }
|
||||
|
||||
//
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
def index = 0
|
||||
|
||||
+7
-7
@@ -93,13 +93,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年10月总量","23年11月总量","23年12月总量","占比","贝壳","链家", "我爱我家","幸福里APP","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-04-01"
|
||||
def date = "2023-05-01"
|
||||
def endDate = "2023-06-01"
|
||||
def newDate = "2023-07-01"
|
||||
def startDate = "2023-10-01"
|
||||
def date = "2023-11-01"
|
||||
def endDate = "2023-12-01"
|
||||
def newDate = "2024-01-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -499,8 +499,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2022-03-01"
|
||||
def endDate = "2022-04-01"
|
||||
def startDate = "2023-08-01"
|
||||
def endDate = "2023-10-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
|
||||
+36
-33
@@ -37,44 +37,47 @@ class ExportAllPgData {
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()
|
||||
// exportBigdataDistrictData()
|
||||
////
|
||||
//// //第三方网站权重表
|
||||
//////
|
||||
////// //第三方网站权重表
|
||||
// exportWebsiteRate();
|
||||
//
|
||||
////
|
||||
// exportGuidedPrice()//随人工改变//
|
||||
// exportCommunityGuidedPrice()//随政策//
|
||||
// //处理小区情况
|
||||
//// //处理小区情况
|
||||
// exportCommunityStaging()//没cityid
|
||||
// exportBigdataCommunity()//
|
||||
//
|
||||
////
|
||||
// exportCommunityBuildType()//
|
||||
//
|
||||
exportThirdPartyPrice2(new_dists)//
|
||||
exportBigdataForsale(new_dists) //现在按updatetime导出
|
||||
////
|
||||
// exportThirdPartyPrice2(new_dists)//
|
||||
// exportBigdataForsale(new_dists) //现在按updatetime导出
|
||||
// exportBigdataDeal(new_dists) //2022-07-01
|
||||
|
||||
|
||||
//计算结果相关
|
||||
//
|
||||
//
|
||||
// //计算结果相关
|
||||
// exportAvmEvaluationGJNB()
|
||||
// exportAVMEvaluationItemGJNB()
|
||||
//
|
||||
////
|
||||
// exportCityPrice()
|
||||
// exportRegionPrice()
|
||||
|
||||
exportCommunityBasePrice()
|
||||
|
||||
exportCommunityPdescribeGj()
|
||||
exportCommCaseDetailGj()
|
||||
|
||||
// exportCommunityBasePrice()
|
||||
// exportCommunityBasePrice1(curMonth)
|
||||
|
||||
// exportCommunityPdescribeGj()
|
||||
// exportCommCaseDetailGj()
|
||||
|
||||
//
|
||||
// exportInOk()
|
||||
exportInOk()
|
||||
////
|
||||
// SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
|
||||
}
|
||||
|
||||
def exportEvaluationRateData() {
|
||||
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h";
|
||||
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from base.t0831_evaluate_rate_gj_h";
|
||||
def allRows = []
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
@@ -102,7 +105,7 @@ class ExportAllPgData {
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'";
|
||||
def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
@@ -133,7 +136,7 @@ class ExportAllPgData {
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'";
|
||||
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from base.t0831_bigdata_district_gj_h where urbn_id='${cityCode}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_admndiv_id
|
||||
@@ -164,7 +167,7 @@ class ExportAllPgData {
|
||||
}
|
||||
|
||||
def exportWebsiteRate(){
|
||||
def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h";
|
||||
def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_website_config_gj_h";
|
||||
def allRows = []
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
@@ -194,7 +197,7 @@ class ExportAllPgData {
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'";
|
||||
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_fsid
|
||||
@@ -227,7 +230,7 @@ class ExportAllPgData {
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}'";
|
||||
def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from base.t0831_community_guide_price_gj_h where urbn_id='${cityCode}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_fsid
|
||||
@@ -307,7 +310,7 @@ class ExportAllPgData {
|
||||
def exportCommunityStaging() {
|
||||
|
||||
def allRows = []
|
||||
def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h";
|
||||
def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_community_staging_gj_h";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_instm_id
|
||||
@@ -330,7 +333,7 @@ class ExportAllPgData {
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'";
|
||||
def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from base.t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
@@ -363,12 +366,12 @@ class ExportAllPgData {
|
||||
def allRows = []
|
||||
if(new_dists) {
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
|
||||
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
|
||||
"urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr} ";
|
||||
def rets1 = getThirdPartyDatas(sql1)
|
||||
allRows.addAll(rets1)
|
||||
|
||||
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
|
||||
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
|
||||
"urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} ";
|
||||
def rets2 = getThirdPartyDatas(sql2)
|
||||
allRows.addAll(rets2)
|
||||
@@ -424,17 +427,17 @@ class ExportAllPgData {
|
||||
if(new_dists){
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
|
||||
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"
|
||||
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"
|
||||
def rets1 = dealData(sql1)
|
||||
|
||||
def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
|
||||
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
|
||||
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
|
||||
def rets2 = dealData(sql2)
|
||||
allRows.addAll(rets1)
|
||||
allRows.addAll(rets2)
|
||||
}else{
|
||||
def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
|
||||
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";
|
||||
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";
|
||||
allRows.addAll(dealData(sql))
|
||||
}
|
||||
}
|
||||
@@ -535,7 +538,7 @@ class ExportAllPgData {
|
||||
if(new_dists){
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
|
||||
"from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}";
|
||||
"from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}";
|
||||
def rets1 = getHouseDatas(sql1)
|
||||
allRows.addAll(rets1)
|
||||
}else{
|
||||
@@ -560,7 +563,7 @@ class ExportAllPgData {
|
||||
if(new_dists){//补充被修区域的数据
|
||||
def newstr = "('"+new_dists.join("','")+"')"
|
||||
def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
|
||||
"from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
|
||||
"from base.t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
|
||||
def rets2 = getHouseDatas(sql2)
|
||||
if(rets2.size()>0){
|
||||
FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true);
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ class ExportPgsqlBaseEvaluation {
|
||||
|
||||
def database = "base"
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
|
||||
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
|
||||
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\"
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-2
@@ -41,11 +41,10 @@ class ExportPgsqlEvaluationResult {
|
||||
exportRegionPrice(curMonth)
|
||||
|
||||
exportCommunityBasePrice(curMonth)
|
||||
// exportCommunityBasePrice1(curMonth)
|
||||
|
||||
exportCommunityPdescribeGj(curMonth)
|
||||
exportCommCaseDetailGj(curMonth)
|
||||
//
|
||||
|
||||
exportInOk()
|
||||
//
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
|
||||
+2
-2
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
|
||||
|
||||
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
|
||||
// cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
// regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
}
|
||||
|
||||
|
||||
+42
@@ -52,6 +52,8 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
exportBigdataBuildingFJW()//
|
||||
exportBigdataCommfeeGj()//
|
||||
|
||||
exportGuidedPrice()//10月10号再同步数据
|
||||
|
||||
exportBigdataCommbuildstruGJ()//
|
||||
exportBigdataCommhousetypeGJ()//
|
||||
exportBigdataCommbuildyearGj()//
|
||||
@@ -188,6 +190,46 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMUNITY_FJW ÉÏ´«FtpÍê³É......"
|
||||
}
|
||||
|
||||
def exportGuidedPrice() {
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_fsid
|
||||
data << r.gujia_bds_id
|
||||
data << r.comm_nm
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.gujia_drc_dt
|
||||
data << r.gujia_bds_drc_prc
|
||||
data << r.gujia_add_psn
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
|
||||
println "GUIDED_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataCommbuildstruGJ(){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
|
||||
@@ -19,10 +19,12 @@ class PocTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new PocTest();
|
||||
task.run();
|
||||
// task.run();
|
||||
// task.export()
|
||||
// task.distance()
|
||||
}
|
||||
|
||||
|
||||
def export(){
|
||||
ExcelUtils.modify("D:\\Êý¾Ý\\ÆäËü\\¹ãÖÝ.xlsx", { row ->
|
||||
def dist = CommonDatautils.findDistrictById(row["B"])
|
||||
@@ -156,7 +158,6 @@ class PocTest {
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
// row['Q'] = dist.name
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['R'] = priceObj.avg_price
|
||||
|
||||
+467
-70
@@ -1,8 +1,12 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
@@ -32,6 +36,7 @@ import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class CommonDatautils {
|
||||
|
||||
@@ -83,8 +88,17 @@ class CommonDatautils {
|
||||
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
public static final DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
|
||||
static def DIRECTIONS = ["¶«±±", "Î÷±±", "¶«ÄÏ", "Î÷ÄÏ", "Äϱ±", "¶«Î÷", "¶«", "ÄÏ", "Î÷", "±±"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
@@ -227,12 +241,11 @@ class CommonDatautils {
|
||||
|
||||
public static def findRegionDistrictCount(def cat, def city, def region){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible", 0);
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("region", region);
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
return cursor.count();
|
||||
return DbUtils.seaweed.count(q);
|
||||
}
|
||||
|
||||
public static def findAllFiledDistrict(def cat, def city){
|
||||
@@ -246,12 +259,13 @@ class CommonDatautils {
|
||||
List<DBObject> dbs = cursor.getResults();
|
||||
return dbs;
|
||||
}
|
||||
|
||||
public static def findAllDistricts(def city, def cat, def fields){
|
||||
|
||||
public static def findAllDistrictsFromMg(def city, def cat, def fields){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible", 0);
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","泰瑞家园")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -260,7 +274,7 @@ class CommonDatautils {
|
||||
}
|
||||
}
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
@@ -298,7 +312,11 @@ class CommonDatautils {
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
if(startDate){
|
||||
q.put("m_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def datequery = new BasicDBObject('$gte', startDate)
|
||||
if(endDate){
|
||||
datequery.append('$lt',endDate)
|
||||
}
|
||||
q.put("m_date", datequery)
|
||||
}
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
@@ -784,10 +802,33 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def findAllDistricts(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistricMaptByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
|
||||
@@ -797,7 +838,7 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
|
||||
def retMap = [:]
|
||||
for(def d : dbsour){
|
||||
retMap.put(d._id.toString(), d)
|
||||
@@ -806,27 +847,27 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","Íò´ï»ª¸®Ò»ÆÚ")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
// public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
//
|
||||
// DBObject q = new BasicDBObject();
|
||||
// q.put("status",0)
|
||||
// q.put("cat", cat);
|
||||
// q.put("city", city);
|
||||
//// q.put("name","万达华府一期")
|
||||
//
|
||||
// DBObject keys = new BasicDBObject();
|
||||
// if(fields){
|
||||
// for(def field : fields){
|
||||
// keys.put(field, 1);
|
||||
// }
|
||||
// }
|
||||
// def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
|
||||
// def rets = []
|
||||
// for(def d : dbsour){
|
||||
// rets << d
|
||||
// }
|
||||
// return rets;
|
||||
// }
|
||||
|
||||
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
|
||||
|
||||
@@ -1731,14 +1772,14 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name, weekend){
|
||||
public static def queryFirstDistrictPrice(city, region, name){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("weekend", weekend);
|
||||
// queryCondition.put("weekend", weekend);
|
||||
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def dbo = null;
|
||||
while(cur.hasNext()) {
|
||||
@@ -2206,6 +2247,21 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryAllDistrictPrice(city, startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put('weekend', startDate);
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rMap = [:];
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
rMap.put(city + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
public static def queryAllDistrictPrice(startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put('weekend', startDate);
|
||||
@@ -2604,6 +2660,7 @@ class CommonDatautils {
|
||||
q.put("cat", cat);
|
||||
q.put('lat2', new BasicDBObject(QueryOperators.LTE, maxLat).append(QueryOperators.GTE, minLat));
|
||||
q.put('lng2', new BasicDBObject(QueryOperators.LTE, maxLng).append(QueryOperators.GTE, minLng));
|
||||
q.put("visible", 0)
|
||||
def cur = getDataStoreConnection().find(q);
|
||||
def ret = [];
|
||||
for(def obj : cur){
|
||||
@@ -3862,42 +3919,6 @@ class CommonDatautils {
|
||||
return avg_price
|
||||
}
|
||||
|
||||
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
@@ -3941,5 +3962,381 @@ class CommonDatautils {
|
||||
return files;
|
||||
}
|
||||
|
||||
static def findLogs(userId, requestUri, startDate, endDate, code){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
|
||||
if(startDate){
|
||||
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
|
||||
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
|
||||
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
|
||||
queryBuilder.must(date_range_cdate);
|
||||
}
|
||||
if(code){
|
||||
queryBuilder.must(QueryBuilders.termQuery("code", code))
|
||||
}
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
return rets.size()
|
||||
}
|
||||
|
||||
static def findLogsBySql(userId, requestUri, startDate, endDate, code){
|
||||
String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' "
|
||||
if(startDate){
|
||||
sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'"
|
||||
}
|
||||
if(code){
|
||||
sql = sql + " and code=${code}"
|
||||
}
|
||||
def total = null
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
return total;
|
||||
}
|
||||
|
||||
static def findNewAmapDetailBySql(city){
|
||||
String sql = "select * from amap_poi where cityname='${city}' and status=0"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
def point = data.location
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
|
||||
rets << ["data_id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location]
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def findAmapDetailBySql(city){
|
||||
String sql = "select * from amap_poi where cityname='${city}' and status=0"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
// if(data.name.contains('华山小区G栋')){
|
||||
def point = data.location
|
||||
// def tag = null
|
||||
// if(data.get("business.keytag")){
|
||||
// tag = data.get("business.keytag")
|
||||
// }
|
||||
def business_area = null
|
||||
if(data.get("business.business_area")){
|
||||
business_area = data.get("business.business_area")
|
||||
}
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
|
||||
rets << ["id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
|
||||
// }
|
||||
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
//用sql有些字段取不出来
|
||||
static def findDetailLogsBySql(userId, requestUri){
|
||||
String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200"
|
||||
|
||||
sql = sql + " order by c_date desc"
|
||||
def total = null
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
def ret = []
|
||||
println data.toString()
|
||||
ret << data.get("parameter_map.city")
|
||||
// ret << data.get("parameter_map.district")
|
||||
// ret << data.get("parameter_map.keyword")
|
||||
// ret << data.get("parameter_map.name")
|
||||
ret << data.get("parameter_map.address")
|
||||
// ret << data.get("parameter_map.size")
|
||||
ret << data.get("msg")
|
||||
ret << data.get("code")
|
||||
ret << data.get("c_date").toString()[0..10]
|
||||
rets << ret
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def findNewAmapDetail(city) {
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city))
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder);
|
||||
def results = []
|
||||
if(rets) {
|
||||
for (int k = 0; k < rets.size(); k++) {
|
||||
def data = rets[k];
|
||||
println data
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
static def findLogs2(userId, requestUri){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
|
||||
|
||||
queryBuilder.mustNot(QueryBuilders.termQuery("code", 200))
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def results = []
|
||||
if(rets) {
|
||||
for (int k = 0; k < rets.size(); k++) {
|
||||
def data = rets[k];
|
||||
println data
|
||||
def ret = []
|
||||
println data.toString()
|
||||
ret << data.parameter_map.city
|
||||
ret << data.parameter_map.district
|
||||
ret << data.parameter_map.name
|
||||
ret << data.parameter_map.address
|
||||
ret << data.msg
|
||||
ret << data.code
|
||||
ret << data.c_date.toString()[0..10]
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
static def httpSaveData(def newUrl,def params){
|
||||
def header = [:]
|
||||
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
|
||||
doPost(newUrl,header, params, true, "utf-8")
|
||||
}
|
||||
|
||||
|
||||
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
try{
|
||||
URL url = new URL(targetUrl);
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setDoOutput(true);
|
||||
conn.setUseCaches(false);
|
||||
conn.setConnectTimeout(5000);
|
||||
conn.setReadTimeout(5000);
|
||||
conn.setRequestProperty("User-Agent", UserAgents.get());
|
||||
String param = null;
|
||||
BufferedWriter mWriter = null;
|
||||
InputStreamReader inStream = null;
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
conn.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
} else {
|
||||
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
|
||||
}
|
||||
|
||||
if (isJson) {
|
||||
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
|
||||
param = JacksonUtil.beanToJson(params)
|
||||
}
|
||||
|
||||
conn.connect();
|
||||
|
||||
//println param.toString()
|
||||
//输出流
|
||||
if (StringUtils.isNotEmpty(param)) {
|
||||
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
|
||||
mWriter.write(param);
|
||||
mWriter.flush();
|
||||
mWriter.close();
|
||||
}
|
||||
if (conn.getHeaderField("Content-Encoding") != null
|
||||
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
|
||||
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
|
||||
} else {
|
||||
inStream = new InputStreamReader(conn.getInputStream(), charset);
|
||||
}
|
||||
char[] bf = new char[1024];
|
||||
int len;
|
||||
while ((len = inStream.read(bf)) != -1) {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取小区周边一公里小区数量
|
||||
* @param city
|
||||
* @param lng
|
||||
* @param lat
|
||||
* @return
|
||||
*/
|
||||
static getDistrictArround(city, lng, lat) {
|
||||
try {
|
||||
def query = new BasicDBObject()
|
||||
query.append("cat", "district")
|
||||
query.append("city", city)
|
||||
query.append("visible", 0)
|
||||
DBObject sdq = new BasicDBObject();
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
|
||||
query.append('$sd', sdq)
|
||||
return DbUtils.seaweed.find(query).toList()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
static def findThirdPartyPrice(def city, def region, def name, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
static def findGXDPrice(def city, region, name){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("source", "GXD")
|
||||
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
|
||||
def price = null
|
||||
for(def d : dbcursor){
|
||||
price = d.price
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
static def findTownRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def fjwcity = null
|
||||
if(d.level == "城市"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(!fjwcity){
|
||||
if(d.city == "市辖区"){
|
||||
fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
d.city = d.province
|
||||
}
|
||||
}
|
||||
}
|
||||
}else if(d.level == "区域"){
|
||||
if(d.administer == "省直辖县级行政区划"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
d.city = d.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(fjwcity){
|
||||
retMap.put(fjwcity, d)
|
||||
}else{
|
||||
println d.toString() +" 格式化不成功!"
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("level","区域")
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.city == "省直辖县级行政区划"){
|
||||
continue
|
||||
}else if(d.city == "市辖区"){
|
||||
def fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}else{
|
||||
def fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
def rets = []
|
||||
|
||||
def ids = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
}
|
||||
println "[${city}] 小区量:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+7
-7
@@ -24,16 +24,16 @@ class LoudongDataCoverBatch extends BaseBatch {
|
||||
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
|
||||
public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
|
||||
public static final def EMAILS = [
|
||||
"duxiaoyu@fangjia.com",
|
||||
// "duxiaoyu@fangjia.com",
|
||||
// "ruanyadong@fangjia.com",
|
||||
// "xiamin@fangjia.com",
|
||||
// "zhutianli@fangjia.com",
|
||||
"joe@fangjia.com",
|
||||
"yukai@fangjia.com",
|
||||
"jiangxiaochun@fangjia.com",
|
||||
"gaojuan@fangjia.com",
|
||||
"shijing@fangjia.com",
|
||||
"xiaohai@fangjia.com",
|
||||
// "joe@fangjia.com",
|
||||
// "yukai@fangjia.com",
|
||||
// "jiangxiaochun@fangjia.com",
|
||||
"gaojuan@fangjia.com"
|
||||
// "shijing@fangjia.com",
|
||||
// "xiaohai@fangjia.com",
|
||||
]
|
||||
|
||||
static def FIELDS_MAP = [
|
||||
|
||||
+2
-2
@@ -27,8 +27,8 @@ class ApiDailyCount extends BaseBatch{
|
||||
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
|
||||
|
||||
def static emails = [
|
||||
'xuqi@fangjia.com',
|
||||
'zongzhimin@fangjia.com'
|
||||
'wangwenyan@fangjia.com',
|
||||
'gaojuan@fangjia.com'
|
||||
]
|
||||
|
||||
static void main(String[] args){
|
||||
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
package com.fangjia.script.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictFacilities2Batch extends BaseBatch {
|
||||
|
||||
public DBCollection onlineColl = MongoUtils.getCollectionByName("fangjia", "district_arround_poi", "mongo.db.fangjia.com")
|
||||
|
||||
public static void main(String[] args) {
|
||||
//小区周边数据更新
|
||||
this.runBatch(new DistrictFacilities2Batch(), args);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def cities = CommonDatautils.findAllCity()
|
||||
for (def city : cities) {
|
||||
def ret = CommonDatautils.getAllDistricts("district", city)
|
||||
def index = 0;
|
||||
for (def dbo : ret) {
|
||||
index++;
|
||||
println index + "/" + ret.size() + " " + dbo.city + " " + dbo.name
|
||||
|
||||
String region = dbo.get('region')
|
||||
String districtName = dbo.get('name')
|
||||
|
||||
if (!dbo.lat2 && !dbo.lng2) {
|
||||
continue;
|
||||
}
|
||||
def poms = findPoms(dbo.lat2 / 10000000000.0, dbo.lng2 / 10000000000.0);
|
||||
DBObject object = new BasicDBObject();
|
||||
object.put("hospital", unique(processType(poms.get('医院'), 1000, null, ['宠物', '动物', '口腔', '牙科', '美容', '齿科', '眼科', '号楼', '住院部', 'Pure', 'Tokushinkai', '假牙制作', '镶牙', '咪', '喵'])))
|
||||
object.put("shop", processType(poms.get('超市'), 1000, ['便民商店', '便利店'], ['杂货店', '烟酒', '爱鑫', '超市便利店']))
|
||||
object.put("university", processType(poms.get('学校'), 1000, ['大学'], ['食堂', '办公室', '德威', '早教', '夕阳红', '公司']))
|
||||
object.put("middle_school", processType(poms.get('学校'), 1000, ['中学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
|
||||
object.put("primary_school", processType(poms.get('学校'), 1000, ['小学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
|
||||
object.put("kindergarten", processType(poms.get('学校'), 1000, ['幼儿园', '托儿所'], ['食堂', '亲子', '办公室', '夕阳红', '公司']))
|
||||
object.put("food_shop", processType(poms.get('餐饮'), 500, ['餐饮服务']))
|
||||
def market = [];
|
||||
if (poms.get('商场')) {
|
||||
market = market + poms.get('商场');
|
||||
}
|
||||
if (poms.get('超市')) {
|
||||
market = market + poms.get('超市');
|
||||
}
|
||||
object.put("supermarket", processType(market, 1000, ['购物中心', "超市", "好又多"], ['公司', '村村寨寨', '伍缘折扣', '杂货店']))
|
||||
object.put("bank", processType(poms.get('银行'), 1000, ['银行'], ['自动提款机']))
|
||||
object.put("atm", processType(poms.get('ATM'), 1000, ['自动提款机', "atm", "ATM"]))
|
||||
updateToDistrictFacilitys(city, region, districtName, object)
|
||||
updateToDistrictFacilitysScore(city, region, districtName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findPoms(def lat, def lng) {
|
||||
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
|
||||
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put('lat', new BasicDBObject(QueryOperators.LTE, map.maxLat).append(QueryOperators.GTE, map.minLat));
|
||||
q.put('lng', new BasicDBObject(QueryOperators.LTE, map.maxLng).append(QueryOperators.GTE, map.minLng));
|
||||
def cursor = POI_COLL.find(q);
|
||||
def resMap = [:];
|
||||
for (def dbo : cursor) {
|
||||
def keyword = dbo.keyword;
|
||||
def details = resMap.get(keyword);
|
||||
if (!details) {
|
||||
details = [];
|
||||
resMap.put(keyword, details);
|
||||
}
|
||||
|
||||
def dMap = [:];
|
||||
dMap.name = dbo.name;
|
||||
dMap.lat = dbo.lat;
|
||||
dMap.lng = dbo.lng;
|
||||
dMap.distance = com.fangjia.script.util.GeoUtils.calculateDistance(lat, lng, dbo.lat, dbo.lng);
|
||||
details << dMap;
|
||||
}
|
||||
|
||||
resMap.each { k, v ->
|
||||
println "[${k}] " + v;
|
||||
}
|
||||
return resMap;
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -75,7 +75,7 @@ class DistrictHouseBatch extends BaseBatch {
|
||||
sellQuery.put('$facet', facet)
|
||||
sellQuery.put('city', city)
|
||||
sellQuery.put('cat',cat)
|
||||
sellQuery.put('visible',0)
|
||||
// sellQuery.put('visible',0)
|
||||
DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0)
|
||||
List<FacetResult> sellFacetResult = sellDSCursor.getFacetResults();
|
||||
if(sellFacetResult){
|
||||
|
||||
+1
-1
@@ -82,7 +82,7 @@ class LngLatEachWeek extends BaseBatch {
|
||||
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
|
||||
jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href)
|
||||
jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href)
|
||||
jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href)
|
||||
// jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href)
|
||||
}
|
||||
|
||||
public static Map<String, Object> getLngLatByAddress(String city, String address){
|
||||
|
||||
+138
-9
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
@@ -38,9 +39,8 @@ class AmapAddressUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new AmapAddressUpdate()
|
||||
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
|
||||
def addr = task.formatAddress("福州", "北三环路201号")
|
||||
println addr
|
||||
def path = "d:\\数据\\小区\\重庆小区地址与高德地图V1.xlsx"
|
||||
|
||||
// task.run(path)
|
||||
// task.similarMatch(path)
|
||||
// task.addressRun(path)
|
||||
@@ -50,11 +50,15 @@ class AmapAddressUpdate {
|
||||
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
|
||||
// task.updateData(path)
|
||||
// task.updateLocation(path)//更新经纬度
|
||||
|
||||
task.matchNoSameRegionDist()
|
||||
}
|
||||
def run(path){
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"
|
||||
def citys = ["重庆"]//CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
|
||||
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度","得分","匹配类型","小区经纬度与高德距离","操作","来源","地址与小区经纬度距离","备注","经纬度是否修改","高德类型"]
|
||||
|
||||
for (def city : citys) {
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
|
||||
def rets = findAmapDetail(disticts)
|
||||
@@ -63,6 +67,17 @@ class AmapAddressUpdate {
|
||||
ExcelUtils.write(path, allrets, "数据")
|
||||
}
|
||||
|
||||
def matchNoSameRegionDist(){
|
||||
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
|
||||
for (def city : citys) {
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address","estate_type2", "addr_alias","lat", "lng"])
|
||||
def rets = findDiffRegionAmapDetail(disticts)
|
||||
ExcelUtils.write("D:\\数据\\小区\\区域有问题小区.xlsx", rets, city)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def amapRun(){
|
||||
def city = "福州市"
|
||||
def datas = findAllAmapByCity(city)
|
||||
@@ -177,7 +192,15 @@ class AmapAddressUpdate {
|
||||
row << formatAddress(d.city, dbo.address)
|
||||
row << dbo.lng
|
||||
row << dbo.lat
|
||||
row << 1
|
||||
row << dbo.match_type
|
||||
if(d.lng && dbo.lng){
|
||||
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
|
||||
row << distance
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
@@ -186,6 +209,8 @@ class AmapAddressUpdate {
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
}
|
||||
println totalCount.get() +' '+row
|
||||
rets << row
|
||||
@@ -201,6 +226,75 @@ class AmapAddressUpdate {
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findDiffRegionAmapDetail(districts){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
rets << ["城市","区域","名称","地址","物业类型","地图"]
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def dbos = getAmapLocations(d.city+"市", d.name)
|
||||
if(!dbos){
|
||||
def newName = d.name
|
||||
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
|
||||
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
|
||||
}
|
||||
dbos = getAmapLocations(d.city+"市", newName)
|
||||
}
|
||||
|
||||
if(dbos){
|
||||
def sameFlag = false
|
||||
for(def dbo : dbos){
|
||||
def newRegion = RegionData.formatRegion(d.city, dbo.region)
|
||||
if(d.region == newRegion){
|
||||
sameFlag = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if(!sameFlag){
|
||||
def row = []
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
row << d.address
|
||||
row << d.estate_type2
|
||||
def str = ""
|
||||
for(def dbo : dbos){
|
||||
def distance = null
|
||||
if(d.lng && dbo.lng){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
|
||||
}
|
||||
str = str + dbo.region+"-"+dbo.name+"-"+dbo.address+"-"+dbo.type+"-"+distance+"\n"
|
||||
}
|
||||
row << str
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
println totalCount.get()+" "+d.city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
def formatAddress(def city, def address){
|
||||
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
|
||||
def address_str = null
|
||||
@@ -267,6 +361,38 @@ class AmapAddressUpdate {
|
||||
}
|
||||
|
||||
|
||||
def getAmapLocations(city, name){
|
||||
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
sqlBuffer.append(" and type like '%住宅%'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def list = []
|
||||
while (rs.next()) {
|
||||
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
|
||||
def mapName = rs.getString("name").replace("·","")
|
||||
if(mapName == name){
|
||||
def map = [:]
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
map.region = rs.getString("adname")
|
||||
map.type = rs.getString("type")
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
list << map
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -451,9 +577,11 @@ class AmapAddressUpdate {
|
||||
}
|
||||
|
||||
def updateLocation(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { row ->
|
||||
|
||||
if(row['R'] && row['R']=="是"){
|
||||
index++
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
if(dist1){
|
||||
def udata = new BasicDBObject()
|
||||
@@ -514,8 +642,9 @@ class AmapAddressUpdate {
|
||||
row['J'] = dbo.lng
|
||||
row['K'] = dbo.lat
|
||||
row['L'] = dbo.score
|
||||
row['M'] = dbo.distance
|
||||
row['S'] = dbo.type
|
||||
row['N'] = dbo.distance
|
||||
row['M'] = dbo.match_type
|
||||
row['T'] = dbo.type
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -539,7 +668,7 @@ class AmapAddressUpdate {
|
||||
def mapName = rs.getString("name").replace("·","")
|
||||
|
||||
println name +" "+ mapName
|
||||
if(rs.getString("adname").contains(region)){
|
||||
if(rs.getString("adname") && rs.getString("adname").contains(region)){
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
|
||||
|
||||
+4
-1
@@ -97,7 +97,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
saveOrUpdateFzDist(districts)
|
||||
|
||||
if(day == 15){
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), endMDate);
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), null);
|
||||
remarkRemoveDist(removeDists)
|
||||
}else{
|
||||
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, startMDate, endMDate);
|
||||
@@ -201,6 +201,9 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def udate = new BasicDBObject()
|
||||
udate.put("status",1)
|
||||
udate.put("m_date", rd.m_date)
|
||||
udate.put("region", rd.region)
|
||||
udate.put("name", rd.name)
|
||||
udate.put("estate_type2", rd.estate_type2)
|
||||
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
|
||||
}
|
||||
}
|
||||
|
||||
+37
-1
@@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def ret = createNewDatas("福州")
|
||||
saveToDB(ret)
|
||||
|
||||
def results = selectLastResultDatas(ret);
|
||||
|
||||
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
}
|
||||
def sb = appendResult(headers, ret)
|
||||
def sb = appendResult(headers, results)
|
||||
DataHandleUtils.outPutFile(fileName, sb, charset);
|
||||
FzFtpUtils.ftpUpload(fileName)
|
||||
//生成加密数据文件并上传
|
||||
FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset);
|
||||
}
|
||||
|
||||
def selectLastResultDatas(def rets){
|
||||
def rows = []
|
||||
for(def r : rets){
|
||||
rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]]
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def appendResult(def headers, def ret){
|
||||
@@ -121,6 +131,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
for(def rMap : districts){
|
||||
def curPrice = 0
|
||||
def dbo = null
|
||||
|
||||
if(curPriceMap.containsKey(rMap.get("_id").toString())){
|
||||
dbo = curPriceMap.get(rMap.get("_id").toString())
|
||||
curPrice = dbo.avg_price
|
||||
@@ -147,6 +158,29 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
m << 0
|
||||
}
|
||||
m << curDate.toInteger();
|
||||
if(curPrice && dbo){
|
||||
if(dbo.value_type == 1){
|
||||
m << "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
m << "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
m << "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
m << "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
m << "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
m << "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
m << "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
m << "无方式"
|
||||
}else{
|
||||
m << ""
|
||||
}
|
||||
}else{
|
||||
m << "无"
|
||||
}
|
||||
result << m
|
||||
}
|
||||
return result
|
||||
@@ -300,6 +334,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
//按comm_id修改,小区名重新赋值,防止区域和小区名修改
|
||||
udata.put("region", r[3])
|
||||
udata.put("name", r[4])
|
||||
udata.put("value_type", r[7])
|
||||
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def o = new BasicDBObject()
|
||||
@@ -312,6 +347,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
o.put("avg_price", r[5].toInteger())
|
||||
o.put("month", r[6].toInteger())
|
||||
o.put("c_date", new Date())
|
||||
o.put("value_type", r[7])
|
||||
FZ_DISTRICT_STAT_COLL.save(o)
|
||||
}
|
||||
}
|
||||
|
||||
+35
-28
@@ -32,6 +32,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
runBatch(new DistrictSellStatBatch(), args);
|
||||
@@ -88,18 +90,21 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
}else{
|
||||
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0)
|
||||
// q.put("name", "翠园楼")
|
||||
if(day == 2){
|
||||
districts = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
|
||||
}else{
|
||||
q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
|
||||
districts = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
|
||||
}
|
||||
}
|
||||
|
||||
println index+" "+city +" "+districts.size()+" "+ (new Date()).toString()
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -1);
|
||||
def endDate = DateUtils.getLastDayOfMonth(new Date(), -1);
|
||||
if(districts.size()>0){
|
||||
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//ÉϸöÔÂÐ¡Çø¾ù¼Û
|
||||
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
|
||||
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
|
||||
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
|
||||
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
|
||||
@@ -135,11 +140,12 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
districts.each{dist->
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def key = dist.region +"##" + dist.name
|
||||
|
||||
def weekendPriceObj = weekendPriceMap.get(key)
|
||||
if(weekendPriceObj){
|
||||
def curPriceObj = curMonthlyPriceMap.get(key)
|
||||
@@ -151,24 +157,19 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
|
||||
}
|
||||
|
||||
def curPrice = null;
|
||||
if(curPriceObj){
|
||||
curPrice = curPriceObj.avg_price
|
||||
}else{
|
||||
curPrice = weekendPriceObj.avg_price
|
||||
}
|
||||
def curPrice = weekendPriceObj.avg_price
|
||||
|
||||
if(weekendPriceObj.value_type!=1){
|
||||
if(dist.city == "福州"){
|
||||
// if(Math.abs(mom)*100>3){
|
||||
// def randMarkup = nextDouble(0, 3)
|
||||
// if(mom < 0){
|
||||
// mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
// }else{
|
||||
// mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
// }
|
||||
// curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
// }
|
||||
if(Math.abs(mom)*100>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(mom < 0){
|
||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
}else{
|
||||
mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
}
|
||||
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
}
|
||||
}else{
|
||||
if(Math.abs(mom)*100>10){
|
||||
def randMarkup = nextDouble(0, 10)
|
||||
@@ -197,7 +198,9 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
data.put("commid", dist._id.toString())
|
||||
data.put("mom", mom)
|
||||
data.put("yoy", yoy)
|
||||
data.put("avg_price", curPrice)
|
||||
data.put("avg_price",curPrice)
|
||||
data.put("value_type", weekendPriceObj.value_type)
|
||||
data.put("credit", weekendPriceObj.credit)
|
||||
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
|
||||
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||
}
|
||||
@@ -209,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
data.put("estate_type", dist.estate_type2)
|
||||
data.put("date", curMonth)
|
||||
data.put("avg_price", curPrice)
|
||||
data.put("value_type", weekendPriceObj.value_type)
|
||||
data.put("credit", weekendPriceObj.credit)
|
||||
data.put("m_date", new Date())
|
||||
data.put("commid", dist._id.toString())
|
||||
@@ -224,10 +228,10 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// };
|
||||
// })
|
||||
};
|
||||
})
|
||||
}
|
||||
// latch.await()
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
@@ -245,11 +249,12 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
|
||||
def findPriceMap(city, weekend){
|
||||
def findPriceMap(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DISTRICT_STAT.find(q)
|
||||
// q.put("district_name","金福花园")
|
||||
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
|
||||
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(city == "福州"){
|
||||
@@ -257,7 +262,9 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
d.avg_price = d.avg_price_2b
|
||||
}
|
||||
}
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
}
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
+65
-27
@@ -22,6 +22,10 @@ class FzDataResultStat {
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
@@ -51,59 +55,67 @@ class FzDataResultStat {
|
||||
})
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
||||
def dists = getFzDistricts("福州", ["status"])
|
||||
def idMaps = [:]
|
||||
for(def d : dists){
|
||||
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
|
||||
}
|
||||
|
||||
def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")))
|
||||
def distkeys = []
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
for(def obj : list1){
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
if(!distkeys.contains(key)){
|
||||
distkeys << key
|
||||
}else{
|
||||
println "重复:"+key
|
||||
}
|
||||
map1.put(key, row)
|
||||
map1.put(key, obj)
|
||||
}
|
||||
|
||||
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
def list2 = findFzDistrictStat(lastMonth)
|
||||
def map2 = [:]
|
||||
for(def s : list2) {
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
map2.put(key, row)
|
||||
for(def obj : list2) {
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
map2.put(key, obj)
|
||||
}
|
||||
def rows = []
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||
map1.each {k,list->
|
||||
def row = []
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
|
||||
def index = 0
|
||||
map1.each {k,obj1->
|
||||
def keys = k.split("##")
|
||||
|
||||
def lastObj = map2.get(k)
|
||||
def lastPrice = null
|
||||
if(lastObj){
|
||||
lastPrice = lastObj[5]
|
||||
}
|
||||
def lastPrice = map2.get(k)
|
||||
|
||||
def markup = null
|
||||
try{
|
||||
if(list && lastPrice && lastPrice.toInteger()>0){
|
||||
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
|
||||
if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){
|
||||
markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger()
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.findDistrictById(list[1])
|
||||
def flag = "有效"
|
||||
|
||||
if(dist && dist.status!=0){
|
||||
flag = "失效"
|
||||
def flag = "无效"
|
||||
if(obj1){
|
||||
flag = idMaps.get(obj1.comm_id)
|
||||
}
|
||||
|
||||
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
|
||||
def markup_abs = null
|
||||
if(markup){
|
||||
markup_abs = Math.abs(markup)
|
||||
}
|
||||
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
|
||||
rows << row
|
||||
index++
|
||||
println row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
@@ -255,4 +267,30 @@ class FzDataResultStat {
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
def findFzDistrictStat(lastMonth){
|
||||
def q = new BasicDBObject()
|
||||
q.put("month", lastMonth)
|
||||
return FZ_DISTRICT_STAT_COLL.find(q).toList()
|
||||
}
|
||||
|
||||
def getFzDistricts(def city, fields){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("cat", "district")
|
||||
def rets = []
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
FZ_DIST_COLL.find(q, keys).each{it->
|
||||
rets << it
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+3
-3
@@ -76,7 +76,7 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
row << dist.name
|
||||
row << o.s_date
|
||||
|
||||
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
|
||||
def zgf_price = getLastThirdPrice(o.xingfuli_esf_price, curPrice)
|
||||
row << zgf_price
|
||||
if(zgf_price){
|
||||
add = true;
|
||||
@@ -217,8 +217,8 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.s_date){
|
||||
retMap.s_date = curDate
|
||||
}
|
||||
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
|
||||
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
|
||||
if(!retMap.xingfuli_esf_price && d.xingfuli_esf_price){
|
||||
retMap.xingfuli_esf_price = d.xingfuli_esf_price
|
||||
}
|
||||
if(!retMap.anjuke_esf_price && d.get("anjuke_esf_price")){
|
||||
retMap.anjuke_esf_price = d.get("anjuke_esf_price")
|
||||
|
||||
+26
-1
@@ -3,6 +3,8 @@ package com.fangjia.script.gj
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
@@ -25,7 +27,30 @@ class AddAvgpriceBatch {
|
||||
static main(args) {
|
||||
def task = new AddAvgpriceBatch();
|
||||
// task.run1();
|
||||
task.test()
|
||||
// task.test()
|
||||
// task.sendPriceCommand()
|
||||
task.sendNewDistPrice()
|
||||
}
|
||||
|
||||
def sendNewDistPrice(){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", "district");
|
||||
q.put("c_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-11-14","yyyy-MM-dd")))
|
||||
def dbsour = DbUtils.seaweed.find(q)
|
||||
for(def d : dbsour){
|
||||
addPrice(d.city,d.region,d.name, d.estate_type2, DateUtils.parseFromTightByPattern('2023-07-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-19','yyyy-MM-dd'))
|
||||
}
|
||||
}
|
||||
|
||||
def sendPriceCommand(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx", { row ->
|
||||
if(!row['P'] && row['T'] && ["别墅","普通住宅","商住两用"].contains(row['T'])){
|
||||
//发重算均价命令
|
||||
println row['K']+" "+row['L']+" "+row['M']
|
||||
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def test(){
|
||||
|
||||
+18
-18
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
@@ -18,25 +19,24 @@ class EsSearchUtils {
|
||||
|
||||
static def findPois(def city, def region, def name){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and adname = '" + region + "'")
|
||||
sqlBuffer.append(" and name = '" + name + "'")
|
||||
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString())
|
||||
def rets = []
|
||||
while (rs.next()) {
|
||||
def m = [:]
|
||||
m.city = rs.getString("cityname")
|
||||
m.region = rs.getString("adname")
|
||||
m.name = rs.getString("name")
|
||||
m.address = rs.getString("address")
|
||||
m.type = rs.getString("type")
|
||||
m.code = rs.getString("typecode")
|
||||
m.lng = rs.getString("location.lon")
|
||||
m.lat = rs.getString("location.lat")
|
||||
rets << m
|
||||
}
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
def m = [:]
|
||||
m.city = data.cityname
|
||||
m.region = data.adname
|
||||
m.name = data.name
|
||||
m.address = data.address
|
||||
m.type = data.type
|
||||
m.code = data.typecode
|
||||
|
||||
def location = data.location
|
||||
m.lng = location.lon
|
||||
m.lat = location.lat
|
||||
rets << m
|
||||
})
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
File file = new File("d:\\Êý¾Ý\\ÃÀÍÅ\\FJW_COMM_20230815.csv");
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def d = it.split("@",-1)
|
||||
if(d[1] && d[1]!='¸£ÖÝ' && d[7] && d[7].isNumber()){
|
||||
rows << d
|
||||
}
|
||||
}
|
||||
println rows.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rows.size())
|
||||
|
||||
for(def i=0;i<rows.size();i++){
|
||||
final def d = rows[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d[1])
|
||||
q.put("region", d[2])
|
||||
q.put("district_name", d[3])
|
||||
q.put("date", 202307)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("avg_price", d[7].toString().toDouble().toInteger())
|
||||
udata.put("m_date", new Date())
|
||||
DISTRICT_SELL_STAT.update(q, new BasicDBObject('$set', udata), false, true)
|
||||
println totalCount.get() +" "+d[1] + " "+d[3]
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
+176
-37
@@ -6,6 +6,7 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.CNNMFilter
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.village.BaiduCoordinateConverter
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import net.sf.json.JSONObject
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
@@ -337,7 +338,7 @@ class OpenApiUtils {
|
||||
lat2 = null
|
||||
lng2 = null
|
||||
}
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+" "+map.estateType+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
|
||||
// }else{
|
||||
@@ -373,15 +374,18 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
|
||||
static def searchBaiDuMap(String address, String region, int cityCode) {
|
||||
static def searchBaiDuMap(String region, String township, String keyword,int cityCode) {
|
||||
def rets = []
|
||||
keyword = keyword.replace("居民委员会","").replace("居委会","").replace("居委会","").replace("居委","").replace("村民委员会","").replace("村村委会","村").replace("村委会","村")
|
||||
def address = keyword
|
||||
try {
|
||||
Thread.sleep(500)
|
||||
long t = System.currentTimeMillis()
|
||||
// String url = "http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1" +
|
||||
// "&qt=con&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&sug=0&l=13" +
|
||||
// "&sug_forward=&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}"
|
||||
String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
|
||||
// String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
|
||||
String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
|
||||
def content = new HttpRequest().get(url, null, "utf-8")
|
||||
JSONObject json = JSONObject.fromObject(content)
|
||||
|
||||
@@ -389,49 +393,58 @@ class OpenApiUtils {
|
||||
for (def loc : json.content) {
|
||||
String name = formatName(address)
|
||||
if(loc.uid) {
|
||||
def bean = [:]
|
||||
JSONObject uidInfo = BaiDuApiUtils.getDetailByUid(loc.uid)
|
||||
JSONObject uidInfo = null;//BaiDuApiUtils.getDetailByUid(loc.uid)
|
||||
if(uidInfo != null) {
|
||||
def baiduRegion = uidInfo.get("result").area
|
||||
def baiduCity = uidInfo.get("result").city
|
||||
rets = createData(rets, uidInfo, loc, region, name, address)
|
||||
}else{
|
||||
def baiduCity = loc.city_name
|
||||
if(baiduCity.contains("市")){
|
||||
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
|
||||
}
|
||||
baiduCity = RegionData.formatCity(baiduCity)
|
||||
def newName = formatName(loc.name)
|
||||
if(baiduRegion.contains(region) && (name== newName || address==newName)){
|
||||
JSONObject result = uidInfo.get("result")
|
||||
JSONObject lo = result.get("location")
|
||||
Double lng2 = lo.get("lng")
|
||||
Double lat2 = lo.get("lat")
|
||||
bean.lng2 = lng2
|
||||
bean.lat2 = lat2
|
||||
if(!baiduCity){
|
||||
baiduCity = loc.history_address
|
||||
if(baiduCity.contains("市")){
|
||||
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
|
||||
}
|
||||
baiduCity = RegionData.formatCity(baiduCity)
|
||||
}
|
||||
if(!baiduCity){
|
||||
baiduCity = loc.area_name
|
||||
if(baiduCity.contains("市")){
|
||||
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
|
||||
}
|
||||
baiduCity = RegionData.formatCity(baiduCity)
|
||||
}
|
||||
|
||||
def baiduRegion = loc.area_name.replace(baiduCity,"")
|
||||
|
||||
keyword = keyword.replace("社区","")
|
||||
def bdName = loc.name.replace("居民委员会","").replace("居委会","").replace("居委","").replace("村民委员会","").replace("村村委会","村").replace("村委会","村").replace("社区","")
|
||||
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(bdName, keyword);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, keyword);
|
||||
if(score1 || score2){
|
||||
def bean = [:]
|
||||
bean.name = loc.name
|
||||
bean.address = loc.addr.replace("[]","")
|
||||
bean.city = baiduCity
|
||||
bean.region = baiduRegion
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
|
||||
def lng = gmap.get("lng")
|
||||
def lat = gmap.get("lat")
|
||||
bean.lng = lng
|
||||
bean.lat = lat
|
||||
if(result.get("detail_info")){
|
||||
bean.estateType = result.get("detail_info").get("tag")
|
||||
if(loc.admin_info){
|
||||
bean.township = loc.admin_info.town_name
|
||||
}
|
||||
|
||||
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
|
||||
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
|
||||
}else{
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
|
||||
rets << bean;
|
||||
def locArrs = BaiduCoordinateConverter.bdMercator2Latlng(loc.diPointX/100.0, loc.diPointY/100.0)
|
||||
if(locArrs){
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0], locArrs[1])
|
||||
bean.lng = gmap.get("lng")
|
||||
bean.lat = gmap.get("lat")
|
||||
}
|
||||
|
||||
bean.estateType = loc.std_tag
|
||||
bean.score = score1 + score2
|
||||
rets << bean;
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+loc.name+"-"+ loc.addr.replace("[]","")+" "+bean.estateType+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
|
||||
if(score1 || score2){
|
||||
rets << bean;
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
|
||||
}else{
|
||||
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
|
||||
}
|
||||
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -444,6 +457,71 @@ class OpenApiUtils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def static createData(def rets, def uidInfo, def loc, region, name, address){
|
||||
def bean = [:]
|
||||
def baiduRegion = uidInfo.get("result").area
|
||||
def baiduCity = uidInfo.get("result").city
|
||||
baiduCity = RegionData.formatCity(baiduCity)
|
||||
def newName = formatName(loc.name)
|
||||
if(baiduRegion.contains(region) && (name== newName || address==newName)){
|
||||
JSONObject result = uidInfo.get("result")
|
||||
JSONObject lo = result.get("location")
|
||||
Double lng2 = lo.get("lng")
|
||||
Double lat2 = lo.get("lat")
|
||||
bean.lng2 = lng2
|
||||
bean.lat2 = lat2
|
||||
bean.name = loc.name
|
||||
bean.address = loc.addr.replace("[]","")
|
||||
bean.city = baiduCity
|
||||
bean.region = baiduRegion
|
||||
bean.township = loc.admin_info.town_name
|
||||
bean.score = 1
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
|
||||
def lng = gmap.get("lng")
|
||||
def lat = gmap.get("lat")
|
||||
bean.lng = lng
|
||||
bean.lat = lat
|
||||
if(result.get("detail_info")){
|
||||
bean.estateType = result.get("detail_info").get("tag")
|
||||
}
|
||||
|
||||
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
|
||||
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
|
||||
}else{
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
|
||||
rets << bean;
|
||||
}
|
||||
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
|
||||
if(score1 || score2){
|
||||
|
||||
JSONObject result = uidInfo.get("result")
|
||||
JSONObject lo = result.get("location")
|
||||
bean.lng2 = lo.get("lng")
|
||||
bean.lat2 = lo.get("lat")
|
||||
bean.name = loc.name
|
||||
bean.address = loc.addr.replace("[]","")
|
||||
bean.city = baiduCity
|
||||
bean.region = baiduRegion
|
||||
bean.township = loc.admin_info.town_name
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(bean.lng2, bean.lat2)
|
||||
bean.lng = gmap.get("lng")
|
||||
bean.lat = gmap.get("lat")
|
||||
if(result.get("detail_info")){
|
||||
bean.estateType = result.get("detail_info").get("tag")
|
||||
}
|
||||
bean.score = score1 + score2
|
||||
rets << bean;
|
||||
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+bean.estateType+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
|
||||
}else{
|
||||
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def static formatName(String name) {
|
||||
if(name.indexOf("¡¤") != -1) {
|
||||
name = name.replaceAll("¡¤", "")
|
||||
@@ -566,9 +644,70 @@ class OpenApiUtils {
|
||||
return null;
|
||||
}
|
||||
|
||||
static def jsHouseCase(city, region, name, propertyTypeCode){
|
||||
def url = "http://localhost:8080/fangjia-open/house/case?city=${city}&district=${region}&buildingName=${name}&propertyTypeCode=${propertyTypeCode}"
|
||||
// println url
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def dto = []
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}else{
|
||||
println url
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def jsHouseCaseByAddress(address, area){
|
||||
address = address.replace(" ","").replace("%","")
|
||||
// def url = "http://open.fangjia.com/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
def url = "http://localhost:8080/fangjia-open/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
if(area){
|
||||
url = url +"&strctureArea="+area
|
||||
}
|
||||
// println url
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def dto = []
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}else{
|
||||
println url
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def houseValueInformation(city, address, area){
|
||||
address = address.replace(" ","").replace("%","")
|
||||
def url = "http://open.fangjia.com/comprehensiveQuery/houseValueInformation?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
if(area){
|
||||
url = url +"&size="+area
|
||||
}
|
||||
// println url
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def dto = []
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}else{
|
||||
println url
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def financialRating(city, address){
|
||||
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
print(url)
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
|
||||
@@ -158,7 +158,6 @@ class PoiLocation {
|
||||
gmap.lat = DataHandleUtils.getDoubleRound(6, gmap.lat)
|
||||
}
|
||||
return location;
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-06-01"
|
||||
static def statDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1),"yyyy-MM-dd")
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
package com.fangjia.script.gj.abc
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class SuzouPocTest {
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new SuzouPocTest()
|
||||
// task.run()
|
||||
task.newRun()
|
||||
}
|
||||
|
||||
def newRun(){
|
||||
println("start:"+new Date().toString())
|
||||
def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls"
|
||||
newMatch(filePath)
|
||||
addEvalutePrice(filePath)
|
||||
addLation(filePath)
|
||||
addAroundPrice(filePath)
|
||||
println("end:"+new Date().toString())
|
||||
|
||||
|
||||
}
|
||||
|
||||
def newMatch(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
if(d['G'] && !d['J']){
|
||||
def city = d['B']
|
||||
def address = d['E']
|
||||
def name = d['G']
|
||||
def region = d['C']
|
||||
if(region && name){
|
||||
address = region + name
|
||||
}
|
||||
|
||||
city = RegionData.formatCity(city)
|
||||
if(region){
|
||||
region = RegionData.formatRegion(city, region)
|
||||
}
|
||||
def obj = CommonDatautils.findDistrict(city, region, name.trim(), "district");
|
||||
if(obj){
|
||||
d['M'] = obj._id.toString()
|
||||
d['N'] = obj.get("city")
|
||||
d['O'] = obj.get("region")
|
||||
d['P'] = obj.get("name")
|
||||
if(obj.lng){
|
||||
d['Q'] = obj.get("lng")/10000000000.0
|
||||
d['R'] = obj.get("lat")/10000000000.0
|
||||
}
|
||||
d['S'] = "直接从库中查询"
|
||||
}else{
|
||||
//新的匹配算法
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
|
||||
if(!jsonObject && !jsonObject.containsKey("result")){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, address, "property", true)
|
||||
}
|
||||
if(!jsonObject && !jsonObject.containsKey("result")){
|
||||
def newAddr = d['E']
|
||||
if(!d['E'].contains(d['C'])){
|
||||
newAddr = d['C'] + d['E']
|
||||
}
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, newAddr, "property", true)
|
||||
}
|
||||
if(!jsonObject.containsKey("result")){
|
||||
//老匹配
|
||||
jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
|
||||
if(jsonObject && jsonObject.credit>0){
|
||||
d['S'] = "老匹配"
|
||||
}
|
||||
}
|
||||
if (jsonObject != null && !jsonObject.containsKey("result")) {
|
||||
d['M'] = jsonObject.get("id")
|
||||
d['N'] = jsonObject.get("city")
|
||||
d['O'] = jsonObject.get("district")
|
||||
d['P'] = jsonObject.get("name")
|
||||
d['Q'] = jsonObject.get("lng")
|
||||
d['R'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addEvalutePrice(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
def city = d['N']
|
||||
if(city && !d['J']){
|
||||
def region = d['O']
|
||||
def name = d['P']
|
||||
def area = d['F']
|
||||
def address = d['E']
|
||||
def floor = null
|
||||
if (d['H'] && d['H'].toString().isNumber()) {
|
||||
floor = d['H'].toString().toDouble().toInteger().toString()
|
||||
}
|
||||
def height = null
|
||||
if (d['J'] && d['J'].toString().isNumber()) {
|
||||
height = d['J'].toString().toDouble().toInteger().toString()
|
||||
}
|
||||
def ret1 = OpenApiClient.addressEvaluation(city, region, name, name, area, floor, height)
|
||||
if (ret1) {
|
||||
d['J'] = ret1.totalPrice
|
||||
d['K'] = ret1.avgPrice
|
||||
}else{
|
||||
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
|
||||
if(priceObj){
|
||||
d['K'] = priceObj.avg_price_2b
|
||||
if(area && area.toString().isNumber() && !area.toString().contains("+")){
|
||||
d['J'] = (int)(area.toString().toDouble()*priceObj.avg_price_2b)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addLation(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
def city = d['B']
|
||||
if(city && !d['J']){
|
||||
def address = d['E']
|
||||
def map = parserByGaode(city, address)
|
||||
if(map){
|
||||
d['Q'] = map.lng
|
||||
d['R'] = map.lat
|
||||
d['T'] = "地址反解析经纬度"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def parserByGaode(def city, def address){
|
||||
def json = GaoDeApiUtils.geo(city, address)
|
||||
println json
|
||||
if(json && json.status=="1"){
|
||||
def ret = json.geocodes
|
||||
if(ret){
|
||||
def str = ret.get(0).location
|
||||
def level = ret.get(0).level
|
||||
|
||||
if(str && ["门牌号","单元号","道路","道路交叉路口","热点商圈","村庄","兴趣点","门址","住宅区"].contains(level)){
|
||||
def arrs = str.split(",")
|
||||
return ["lng":arrs[0], "lat":arrs[1]]
|
||||
}else{
|
||||
println address +" "+level
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addAroundPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['J'] && d['Q']){
|
||||
def city = RegionData.formatCity(d['B'])
|
||||
def dist = new BasicDBObject()
|
||||
dist.city = city
|
||||
dist.lng = d['Q'].toString().toDouble()*10000000000L
|
||||
dist.lat = d['R'].toString().toDouble()*10000000000L
|
||||
dist.estateType = "普通住宅"//d['H']
|
||||
if(dist.lat) {
|
||||
List<DBObject> list = DbUtils.findAroundDistrictByRadius(dist, 1000)
|
||||
if (list) {
|
||||
list = sortByDistance(dist, list);
|
||||
if (list && list.size() > 10) {
|
||||
list = list[0..9]
|
||||
}
|
||||
if (list) {
|
||||
def prices = []
|
||||
for (def s : list) {
|
||||
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
|
||||
if (priceObj) {
|
||||
prices << priceObj.avg_price_2b
|
||||
}
|
||||
}
|
||||
if (prices) {
|
||||
def arround_price = (int) (prices.sum() / prices.size())
|
||||
d['k'] = arround_price
|
||||
def area = d["F"]
|
||||
if(area && area.toString().isNumber() && !area.toString().contains("+")){
|
||||
d['J'] = (int)(area.toString().toDouble()*arround_price)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def sortByDistance(d, list){
|
||||
def newList = []
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
newList << nb
|
||||
}
|
||||
//排序
|
||||
Collections.sort(newList, new Comparator() {
|
||||
@Override
|
||||
int compare(Object o1, Object o2) {
|
||||
if(o1.distance > o2.distance)
|
||||
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
|
||||
if(o1.distance == o2.distance)
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
|
||||
return newList;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def path = "D:\\数据\\江苏农行\\江苏农1062条(总).xlsx"
|
||||
addressEvalute(path)
|
||||
// addFjwTobPrice(path)
|
||||
// addThirdPrice(path)
|
||||
// margeExcel(path)
|
||||
}
|
||||
|
||||
|
||||
def addressEvalute(path){
|
||||
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
|
||||
if(!d['O'] && d['H']){
|
||||
def city = d['D']
|
||||
def region = d['E']
|
||||
def address = d['G']
|
||||
def name= d['H']
|
||||
def area = d['I']
|
||||
def floor = null
|
||||
if( d['J'] && d['J'].toString().isNumber()){
|
||||
floor = d['J'].toString().toDouble().toInteger().toString()
|
||||
}
|
||||
def height = null
|
||||
if(d['K'] && d['K'].toString().isNumber()){
|
||||
height = d['K'].toString().toDouble().toInteger().toString()
|
||||
}
|
||||
def ret1 = OpenApiClient.addressEvaluation(city, region, address, name, area, floor, height)
|
||||
if(ret1){
|
||||
d['O']= ret1.city
|
||||
d['P'] = ret1.district
|
||||
d['Q'] = ret1.name
|
||||
d['R'] = ret1.totalPrice
|
||||
d['S'] = ret1.avgPrice
|
||||
}
|
||||
|
||||
def ret2 = OpenApiClient.addressEvaluation(city, region, address, name, area, null, null)
|
||||
if(ret2){
|
||||
d['T'] = ret2.totalPrice
|
||||
d['U'] = ret2.avgPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addFjwTobPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['O']){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(d['O'], d['P'], d['Q'])
|
||||
if(priceObj){
|
||||
d['V'] = priceObj.avg_price
|
||||
d['W'] = priceObj.avg_price_2b
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addThirdPrice(path) {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if (d['O'] && !d['V']) {
|
||||
def dist = CommonDatautils.findDistrict(d['O'], d['P'], d['Q'], "district")
|
||||
if (dist) {
|
||||
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
d['V'] = priceObj.avg_price
|
||||
d['W'] = priceObj.avg_price_2b
|
||||
}
|
||||
|
||||
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310)
|
||||
def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name)
|
||||
if (priceMap) {
|
||||
if (priceMap.containsKey("lianjia_esf_price")) {
|
||||
d['X'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if (priceMap.containsKey("beike_esf_price")) {
|
||||
d['Y'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if (priceMap.containsKey("anjuke_esf_price")) {
|
||||
d['Z'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
def cityId = cityCodeDataMap.get(dist.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
|
||||
if (map) {
|
||||
d['AA'] = map.avg_price
|
||||
}
|
||||
def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01')
|
||||
if (tradePrice) {
|
||||
d['AB'] = tradePrice.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def margeExcel(path){
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\(合并)江苏农匹配不上149条-.xlsx", 0, 1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
map.put(d['G'], d['O'])
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农1062条.xlsx",{d->
|
||||
def commId = map.get(d['G'])
|
||||
def dist = CommonDatautils.findDistrictById(commId)
|
||||
if(dist){
|
||||
d['O'] = dist.city
|
||||
d['P'] = dist.region
|
||||
d['Q'] = dist.name
|
||||
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
d['V'] = priceObj.avg_price
|
||||
d['W'] = priceObj.avg_price_2b
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+36
-2
@@ -230,12 +230,46 @@ class CcbCodeUtils {
|
||||
}
|
||||
return "11"
|
||||
}
|
||||
//室内结构
|
||||
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
|
||||
|
||||
static def getHouseStructureCode(){
|
||||
return null
|
||||
}
|
||||
|
||||
//室内结构
|
||||
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
|
||||
static def getRoomStructureCode(room_structure){
|
||||
if(room_structure){
|
||||
if(room_structure.contains("平层")){
|
||||
return "11"
|
||||
}else if(room_structure.contains("错层")){
|
||||
return "12"
|
||||
}else if(room_structure.contains("复式")){
|
||||
return "13"
|
||||
}else if(room_structure.contains("阁楼")){
|
||||
return "14"
|
||||
}else if(room_structure.contains("团结户")){
|
||||
return "15"
|
||||
}else if(room_structure.contains("老式房")){
|
||||
return "16"
|
||||
}else if(room_structure.contains("地下室")){
|
||||
return "17"
|
||||
}else if(room_structure.contains("带阁楼")){
|
||||
return "21"
|
||||
}else if(room_structure.contains("带车库")){
|
||||
return "22"
|
||||
}else if(room_structure.contains("带车位")){
|
||||
return "23"
|
||||
}else if(room_structure.contains("带地下室")){
|
||||
return "24"
|
||||
}else if(room_structure.contains("带储藏室")){
|
||||
return "25"
|
||||
}else if(room_structure.contains("跃层")){
|
||||
return "18"
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//租赁方式 整租:0 合租:1 转租:2
|
||||
static def getRentTypeCode(def rentType){
|
||||
if(rentType){
|
||||
|
||||
@@ -90,4 +90,53 @@ class FileUtils {
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){
|
||||
File file = new File(fileName);
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
if(headers){
|
||||
sb.append(headers.join(splitChar))
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), splitChar))
|
||||
sb.append("\n");
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
package com.fangjia.script.gj.chongqin
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 重庆poc测试-2023版
|
||||
*/
|
||||
class PocTest2023 {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new PocTest2023();
|
||||
task.run();
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
// addEvaluationPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试2.xlsx")
|
||||
// addressEvalute("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx")
|
||||
// structure("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx") //九级地址拆分
|
||||
// addThirdPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx")
|
||||
|
||||
// baseInfo("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx")
|
||||
// baseInfoByName("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx")
|
||||
|
||||
|
||||
distScore("D:\\数据\\重庆农商行2023\\楼盘评级样本.xlsx")
|
||||
}
|
||||
|
||||
def addEvaluationPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{ d->
|
||||
|
||||
if(d['E']){
|
||||
def city = d['B']
|
||||
def region = d['C']
|
||||
def address = d['E']
|
||||
def area = d['D']
|
||||
|
||||
//旧地址估值
|
||||
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
|
||||
if(ret1){
|
||||
d['N']= ret1.city
|
||||
d['O'] = ret1.district
|
||||
d['P'] = ret1.name
|
||||
d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0)
|
||||
d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0)
|
||||
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
|
||||
if(dist){
|
||||
d['Q'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addressEvalute(path){
|
||||
|
||||
ExcelUtils.multiThreadingModify(path,{ d->
|
||||
|
||||
if(!d['N'] && d['E']){
|
||||
def city = d['B']
|
||||
def region = d['C']
|
||||
def address = d['E']
|
||||
def area = d['D']
|
||||
|
||||
//旧地址估值
|
||||
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
|
||||
if(ret1){
|
||||
d['N']= ret1.city
|
||||
d['O'] = ret1.district
|
||||
d['P'] = ret1.name
|
||||
d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0)
|
||||
d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0)
|
||||
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
|
||||
if(dist){
|
||||
d['Q'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
|
||||
//新的匹配算法
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
d['R'] = jsonObject.get("city")
|
||||
d['S'] = jsonObject.get("district")
|
||||
d['T'] = jsonObject.get("name")
|
||||
d['U'] = jsonObject.get("id")
|
||||
}
|
||||
|
||||
if( d['Q'] && d['U'] && d['U']==d['Q']){
|
||||
d['V'] = "T"
|
||||
}else{
|
||||
d['V'] = "F"
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def structure(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile,{ d->
|
||||
if(d['E']){
|
||||
def jsonObject = FangjiaClientUtils.newStructureByLevel9(d['B'], d['E'])
|
||||
if(jsonObject != null && jsonObject.msgCode==200 && jsonObject.result) {
|
||||
def ret = jsonObject.result
|
||||
def str = ""
|
||||
if(ret.township){
|
||||
str = ret.township
|
||||
}
|
||||
if(ret.street){
|
||||
str = str + ret.street
|
||||
}
|
||||
d['H'] = str
|
||||
d['I'] = ret.name
|
||||
d['J'] = ret.buildingNumber
|
||||
d['K'] = ret.unitNumber
|
||||
d['L'] = ret.floor
|
||||
d['M'] = ret.roomNumber
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// def addAuthor(inputfile){
|
||||
// def authors = ["曹","张", "陈", "许", "金玉"]
|
||||
// ExcelUtils.modify(inputfile, { d ->
|
||||
// if(d['V'] && d['V'] == "F"){
|
||||
// for(def name : authors){
|
||||
// d['W'] = name
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
|
||||
def baseInfo(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { d ->
|
||||
if(d['E']){
|
||||
def city = d['B']
|
||||
def region = d['C']
|
||||
def address = d['E']
|
||||
def area = d['D']
|
||||
|
||||
println address
|
||||
//旧地址估值
|
||||
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
|
||||
if(ret1){
|
||||
d['R']= ret1.city
|
||||
d['S'] = ret1.district
|
||||
d['F'] = ret1.name
|
||||
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
|
||||
if(dist){
|
||||
d['T'] = dist._id.toString()
|
||||
d['G'] = dist.estate_company
|
||||
d['H'] = dist.developer
|
||||
d['I'] = dist.cubage_percent
|
||||
d['J'] = dist.virescence_percent
|
||||
d['K'] = dist.all_area
|
||||
d['L'] = dist.loudong_count
|
||||
d['M'] = dist.household_count
|
||||
d['N'] = dist.estate_charge
|
||||
if(dist.open_date){
|
||||
d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
if(dist.is_elevator){
|
||||
d['P'] = "有"
|
||||
}else{
|
||||
d['P'] = "无"
|
||||
}
|
||||
|
||||
def newCity = RegionData.formatCity(city)
|
||||
def obj = findPaiMai(newCity, address)
|
||||
if(obj){
|
||||
d['Q'] = "是"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findPaiMai(city, address){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def or = new BasicDBList()
|
||||
or.add(new BasicDBObject("title", new BasicDBObject('$regex', address)))
|
||||
or.add(new BasicDBObject("address", new BasicDBObject('$regex', address)))
|
||||
q.append('$or', or)
|
||||
def obj = paimai43_COLL.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def baseInfoByName(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { d ->
|
||||
if(d['F']) {
|
||||
def city = d['B']
|
||||
def address = d['E']
|
||||
def dist = CommonDatautils.findDistrict(d['R'], d['S'], d['F'], "district")
|
||||
if (dist) {
|
||||
d['T'] = dist._id.toString()
|
||||
d['G'] = dist.estate_company
|
||||
d['H'] = dist.developer
|
||||
d['I'] = dist.cubage_percent
|
||||
d['J'] = dist.virescence_percent
|
||||
d['K'] = dist.all_area
|
||||
d['L'] = dist.loudong_count
|
||||
d['M'] = dist.household_count
|
||||
d['N'] = dist.estate_charge
|
||||
if (dist.open_date) {
|
||||
d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
if (dist.is_elevator) {
|
||||
d['P'] = "有"
|
||||
} else {
|
||||
d['P'] = "无"
|
||||
}
|
||||
|
||||
def newCity = RegionData.formatCity(city)
|
||||
def obj = findPaiMai(newCity, address)
|
||||
if (obj) {
|
||||
d['Q'] = "是"
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def distScore(inputfile) {
|
||||
ExcelUtils.multiThreadingModify(inputfile, { d ->
|
||||
if (d['F']) {
|
||||
def obj = getDistrictRank(d['R'], d['S'], d['F'])
|
||||
if(obj){
|
||||
def score = obj.city_rank*100/obj.city_rank_total
|
||||
score = DataUtil.getRound(score)
|
||||
d['G'] = 100-score
|
||||
d['H'] = obj.city_rank
|
||||
d['I'] = obj.city_rank_total
|
||||
d['J'] = obj.avg_price
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def getDistrictRank(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name",name)
|
||||
return DISTRICT_ANALY_DATA_COLL.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path) {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if (d['P']) {
|
||||
def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
|
||||
if (dist) {
|
||||
|
||||
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
// if(priceObj){
|
||||
// d['V'] = priceObj.avg_price
|
||||
// d['W'] = priceObj.avg_price_2b
|
||||
// }
|
||||
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310)
|
||||
def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name)
|
||||
if (priceMap) {
|
||||
if (priceMap.containsKey("lianjia_esf_price")) {
|
||||
d['X'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if (priceMap.containsKey("beike_esf_price")) {
|
||||
d['Y'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if (priceMap.containsKey("anjuke_esf_price")) {
|
||||
d['Z'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
def cityId = cityCodeDataMap.get(dist.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
|
||||
if (map) {
|
||||
d['AA'] = map.avg_price
|
||||
}
|
||||
def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01')
|
||||
if (tradePrice) {
|
||||
d['AB'] = tradePrice.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
+52
-7
@@ -1,19 +1,19 @@
|
||||
package com.fangjia.script.gj
|
||||
package com.fangjia.script.gj.city
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class CityStatExport {
|
||||
|
||||
@@ -28,8 +28,41 @@ class CityStatExport {
|
||||
def task = new CityStatExport();
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.regionStat()
|
||||
// task.regionStat()
|
||||
// task.stat()
|
||||
// task.houseStat()
|
||||
task.addFieldStat()
|
||||
}
|
||||
|
||||
def addFieldStat(){
|
||||
def fileName = "D:\\数据\\fjw.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(!d['B']){
|
||||
def city = RegionData.formatCity(d['A'])
|
||||
if(city){
|
||||
def cityInfoDto = CommonDatautils.findCityInfo(city)
|
||||
if(cityInfoDto){
|
||||
d['B'] = cityInfoDto.existing_district_count
|
||||
def detailMap = cityInfoDto.district_detail
|
||||
// d['D'] = detailMap.address.toDouble()*100
|
||||
// d['E'] = 100
|
||||
d['D'] = cityInfoDto.price_coverage.toDouble()*100
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},1)
|
||||
|
||||
}
|
||||
|
||||
def houseStat(){
|
||||
def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def houseCount = findHouseCount(d['F'], 20230201)
|
||||
// d['H'] = houseCount
|
||||
d['J'] = findHouse(d['F'])
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def stat(){
|
||||
@@ -47,7 +80,7 @@ class CityStatExport {
|
||||
// def citys = CommonDatautils.findAllCity();
|
||||
|
||||
|
||||
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"]
|
||||
def ret = []
|
||||
ret << ["城市","小区总量"]
|
||||
// ret << ["城市","楼栋总量", "房号总量"]
|
||||
@@ -78,7 +111,7 @@ class CityStatExport {
|
||||
ret << m
|
||||
println m
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
|
||||
ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3")
|
||||
|
||||
}
|
||||
|
||||
@@ -124,7 +157,7 @@ class CityStatExport {
|
||||
def findHouseCount(def city, def startDate){
|
||||
def total = 0
|
||||
try{
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
|
||||
String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
@@ -136,6 +169,18 @@ class CityStatExport {
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
def findHouse(city) {
|
||||
int startDate = 20230201
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
|
||||
.must(date_range)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
return houses.size()
|
||||
}
|
||||
|
||||
def findLoudongCount(String city) {
|
||||
return LOUDONG.count(new BasicDBObject('city', city))
|
||||
}
|
||||
+61
-16
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.commStat
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
@@ -27,7 +28,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
|
||||
|
||||
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
|
||||
//for(def city : citys){
|
||||
@@ -308,21 +309,65 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
|
||||
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
|
||||
|
||||
//区域级统计
|
||||
def city = "合肥"
|
||||
def regions = CommonDatautils.getCityRegions(city)
|
||||
def rows = []
|
||||
for(def region : regions){
|
||||
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
|
||||
def alias_count = 0
|
||||
def addr_alias_count = 0
|
||||
for(def r : districts){
|
||||
if(r.alias && r.alias.size()>0){
|
||||
alias_count ++
|
||||
//def city = "合肥"
|
||||
//def regions = CommonDatautils.getCityRegions(city)
|
||||
//def rows = []
|
||||
//for(def region : regions){
|
||||
// def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
|
||||
// def alias_count = 0
|
||||
// def addr_alias_count = 0
|
||||
// for(def r : districts){
|
||||
// if(r.alias && r.alias.size()>0){
|
||||
// alias_count ++
|
||||
// }
|
||||
// if(r.addr_alias && r.addr_alias.size()>0){
|
||||
// addr_alias_count++
|
||||
// }
|
||||
// }
|
||||
// rows << [region, districts.size(), alias_count, addr_alias_count]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\区县楼盘量统计.xlsx",{d->
|
||||
if(d['C'] && !d['I']){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
if(!city){
|
||||
city = RegionData.formatCity(d['B'])
|
||||
}
|
||||
if(r.addr_alias && r.addr_alias.size()>0){
|
||||
addr_alias_count++
|
||||
if(city && d['D']){
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(region){
|
||||
def count = CommonDatautils.findRegionDistrictCount("district", city, region)
|
||||
println city +" "+ region+" "+count
|
||||
d['I'] = city
|
||||
d['J'] = region
|
||||
d['K'] = count
|
||||
}
|
||||
// def dists = CommonDatautils.findAllDistricts(d['C'], "district", ["city","region","name"])
|
||||
//d['B'] = dists.size()
|
||||
}
|
||||
}
|
||||
rows << [region, districts.size(), alias_count, addr_alias_count]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city",d['B'])
|
||||
// q.put("district","")
|
||||
// def dbo = mca_administrative_division_coll.findOne(q)
|
||||
// if(!dbo){
|
||||
// q.put("city", "省直辖县级行政区划")
|
||||
// q.put("district", d['B'])
|
||||
// dbo = mca_administrative_division_coll.findOne(q)
|
||||
// }
|
||||
// if(dbo){
|
||||
// d['A']=dbo.province
|
||||
// }
|
||||
})
|
||||
|
||||
//def citys = ["北京","上海","重庆","天津","石家庄","唐山","秦皇岛","邯郸","邢台","保定","张家口","承德","沧州","廊坊","衡水","太原","大同","阳泉","长治","晋城","朔州","晋中","运城","忻州","临汾","吕梁","呼和浩特","包头","乌海","赤峰","通辽","鄂尔多斯","呼伦贝尔","巴彦淖尔","乌兰察布","兴安盟","锡林郭勒盟","阿拉善盟","沈阳","大连","鞍山","抚顺","本溪","丹东","锦州","营口","阜新","辽阳","盘锦","铁岭","朝阳","葫芦岛","长春","吉林","四平","辽源","通化","白山","松原","白城","延边","哈尔滨","齐齐哈尔","鸡西","鹤岗","双鸭山","大庆","伊春","佳木斯","七台河","牡丹江","黑河","绥化","大兴安岭","南京","无锡","徐州","常州","苏州","南通","连云港","淮安","盐城","扬州","镇江","泰州","宿迁","杭州","宁波","温州","嘉兴","湖州","绍兴","金华","衢州","舟山","台州","丽水","合肥","芜湖","蚌埠","淮南","马鞍山","淮北","铜陵","安庆","黄山","滁州","阜阳","宿州","六安","亳州","池州","宣城","福州","厦门","莆田","三明","泉州","漳州","南平","龙岩","宁德","南昌","景德镇","萍乡","九江","新余","鹰潭","赣州","吉安","宜春","抚州","上饶","济南","青岛","淄博","枣庄","东营","烟台","潍坊","济宁","泰安","威海","日照","临沂","德州","聊城","滨州","菏泽","郑州","开封","洛阳","平顶山","安阳","鹤壁","新乡","焦作","濮阳","许昌","漯河","三门峡","南阳","商丘","信阳","周口","驻马店","武汉","黄石","十堰","宜昌","襄阳","鄂州","荆门","孝感","荆州","黄冈","咸宁","随州","恩施","长沙","株洲","湘潭","衡阳","邵阳","岳阳","常德","张家界","益阳","郴州","永州","怀化","娄底","湘西","广州","韶关","深圳","珠海","汕头","佛山","江门","湛江","茂名","肇庆","惠州","梅州","汕尾","河源","阳江","清远","东莞","中山","潮州","揭阳","云浮","南宁","柳州","桂林","梧州","北海","防城港","钦州","贵港","玉林","百色","贺州","河池","来宾","崇左","海口","三亚","三沙","儋州","成都","自贡","攀枝花","泸州","德阳","绵阳","广元","遂宁","内江","乐山","南充","眉山","宜宾","广安","达州","雅安","巴中","资阳","阿坝州","甘孜州","凉山州","贵阳","六盘水","遵义","安顺","毕节","铜仁","黔西南","黔东南","黔南","昆明","曲靖","玉溪","保山","昭通","丽江","普洱","临沧","楚雄","红河","文山州","西双版纳州","大理","德宏州","怒江州","迪庆州","拉萨","日喀则","昌都","林芝","山南","那曲","阿里","西安","铜川","宝鸡","咸阳","渭南","延安","汉中","榆林","安康","商洛","兰州","嘉峪关","金昌","白银","天水","武威","张掖","平凉","酒泉","庆阳","定西","陇南","临夏","甘南","西宁","海东","海北州","黄南州","海南州","果洛州","玉树州","海西州","银川","石嘴山","吴忠","固原","中卫","乌鲁木齐","克拉玛依","吐鲁番","哈密","昌吉","博尔塔拉","巴音郭楞","阿克苏","克孜勒苏","喀什","和田","伊犁","塔城","阿勒泰","济源","仙桃","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","石河子","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉","昆玉"]
|
||||
//
|
||||
//def fjwCitys = CommonDatautils.findAllCity()
|
||||
//for(def fjw_city : fjwCitys){
|
||||
// if(!citys.contains(fjw_city)){
|
||||
// println fjw_city
|
||||
// }
|
||||
//}
|
||||
@@ -14,12 +14,12 @@ class FieldStat {
|
||||
static main(args) {
|
||||
def task = new FieldStat();
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.offircRun()
|
||||
task.distRun()
|
||||
task.shopRun()
|
||||
// task.distRun()
|
||||
// task.shopRun()
|
||||
}
|
||||
|
||||
def distRun(){
|
||||
@@ -203,11 +203,15 @@ class FieldStat {
|
||||
alias_total = alias_total + alias.size()
|
||||
}
|
||||
}
|
||||
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
|
||||
def precent = 0
|
||||
if(districts){
|
||||
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
|
||||
}
|
||||
|
||||
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
|
||||
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
|
||||
ExcelUtils.write("d:\\数据\\城市别名统计.xlsx", rets, "统计")
|
||||
// println dist_total
|
||||
// println alias_total
|
||||
// println addr_alias_total
|
||||
|
||||
+29
-15
@@ -5,24 +5,38 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
//
|
||||
//ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
|
||||
// def opt = d['J']
|
||||
// if(opt && opt.contains("别名")){
|
||||
// def dist = CommonDatautils.findDistrictById(d['H'])
|
||||
// if(dist){
|
||||
// def alias = []
|
||||
// if(dist.alias){
|
||||
// alias = dist.alias
|
||||
// }
|
||||
// if(!alias.contains(d['C'])){
|
||||
// alias << d['C']
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("alias", alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\厦门\\厦门户数(曹和付)-结果.xlsx",{d->
|
||||
def opt = d['J']
|
||||
if(opt && opt.contains("别名")){
|
||||
def dist = CommonDatautils.findDistrictById(d['H'])
|
||||
if(opt && opt.toString().toDouble().toInteger()==1){
|
||||
def dist = CommonDatautils.findDistrict('厦门',d['G'],d['H'], "district")
|
||||
if(dist){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
alias = dist.alias
|
||||
}
|
||||
if(!alias.contains(d['C'])){
|
||||
alias << d['C']
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("household_count", d['F'].toString().toDouble().toInteger())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
|
||||
+429
-3
@@ -5,8 +5,10 @@ import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -22,12 +24,177 @@ class ApiTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new ApiTest()
|
||||
// task.rankRunTest()
|
||||
// task.similarRunTest()
|
||||
task.loupanLevelTest()
|
||||
// task.rankRunTest()//小区排名接口测试
|
||||
// task.similarRunTest()//相似房源接口测试
|
||||
// task.loupanLevelTest()//楼盘评级接口测试
|
||||
|
||||
// task.jsAbcHouseCastTest()//北交所房源接口测试 小区
|
||||
// task.arroundFacilities()//北交所周边poi 小区
|
||||
|
||||
|
||||
// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价
|
||||
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
|
||||
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
|
||||
// task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
|
||||
task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
|
||||
|
||||
//五级地址标准化
|
||||
// task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
|
||||
// task.nineLevel("D:\\数据\\地数\\地址九级标准化-测试用例.xlsx")
|
||||
}
|
||||
|
||||
def addrStructure(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['B']){
|
||||
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
|
||||
if(jsonObject) {
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['C'] = jsonObject.get("province")
|
||||
}
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['D'] = jsonObject.get("city")
|
||||
}
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['E'] = jsonObject.get("district")
|
||||
}
|
||||
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
|
||||
d['F'] = jsonObject.get("township")
|
||||
}
|
||||
if(jsonObject.get("address") && jsonObject.get("address")!="null"){
|
||||
d['G'] = jsonObject.get("address")
|
||||
}
|
||||
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
|
||||
d['H'] = jsonObject.get("provinceCode")
|
||||
}
|
||||
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
|
||||
d['I'] = jsonObject.get("cityCode")
|
||||
}
|
||||
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
|
||||
d['J'] = jsonObject.get("districtCode")
|
||||
}
|
||||
}
|
||||
}
|
||||
},7);
|
||||
}
|
||||
|
||||
|
||||
def nineLevel(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['B']){
|
||||
def obj = FangjiaClientUtils.formatNinelevel(d['A'], d['B']);
|
||||
if(obj) {
|
||||
def jsonObject = obj.get("result")
|
||||
|
||||
if(jsonObject){
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['C'] = jsonObject.get("province")
|
||||
}
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['D'] = jsonObject.get("city")
|
||||
}
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['E'] = jsonObject.get("district")
|
||||
}
|
||||
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
|
||||
d['F'] = jsonObject.get("township")
|
||||
}
|
||||
if(jsonObject.get("formatAddress") && jsonObject.get("formatAddress")!="null"){
|
||||
d['G'] = jsonObject.get("formatAddress")
|
||||
}
|
||||
|
||||
if(jsonObject.get("community") && jsonObject.get("community")!="null"){
|
||||
d['H'] = jsonObject.get("community")
|
||||
}
|
||||
|
||||
if(jsonObject.get("name") && jsonObject.get("name")!="null"){
|
||||
d['I'] = jsonObject.get("name")
|
||||
}
|
||||
|
||||
if(jsonObject.get("street") && jsonObject.get("street")!="null"){
|
||||
d['J'] = jsonObject.get("street")
|
||||
}
|
||||
|
||||
if(jsonObject.get("streetNumber") && jsonObject.get("streetNumber")!="null"){
|
||||
d['K'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
|
||||
|
||||
if(jsonObject.get("buildingNumber") && jsonObject.get("buildingNumber")!="null"){
|
||||
d['L'] = jsonObject.get("buildingNumber")
|
||||
}
|
||||
|
||||
if(jsonObject.get("unitNumber") && jsonObject.get("unitNumber")!="null"){
|
||||
d['M'] = jsonObject.get("unitNumber")
|
||||
}
|
||||
|
||||
if(jsonObject.get("roomNumber") && jsonObject.get("roomNumber")!="null"){
|
||||
d['N'] = jsonObject.get("roomNumber")
|
||||
}
|
||||
|
||||
if(jsonObject.get("floor") && jsonObject.get("floor")!="null"){
|
||||
d['O'] = jsonObject.get("floor")
|
||||
}
|
||||
|
||||
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
|
||||
d['P'] = jsonObject.get("provinceCode")
|
||||
}
|
||||
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
|
||||
d['Q'] = jsonObject.get("cityCode")
|
||||
}
|
||||
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
|
||||
d['R'] = jsonObject.get("districtCode")
|
||||
}
|
||||
|
||||
def map = jsonObject.get("addressAssessment")
|
||||
if(map){
|
||||
d['S'] = map.get("msg")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def addressEvaluate(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['B']){
|
||||
def ret1 = OpenApiClient.addressEvaluation(d['A'], null, d['B'], null, d['C'].toString().toDouble().toString(), null, null)
|
||||
if(ret1){
|
||||
d['G']= ret1.city
|
||||
d['H'] = ret1.district
|
||||
d['I'] = ret1.name
|
||||
d['J'] = ret1.totalPrice
|
||||
d['K'] = ret1.avgPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addressAvgPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['B']){
|
||||
println "[match] city:"+d['A']+" address:"+d['B']
|
||||
def jsonObject = FangjiaClientUtils.newMatchPrice(d['A'], d['B'], "property", false)
|
||||
|
||||
if(jsonObject && jsonObject.containsKey("result")) {
|
||||
def resultMap = jsonObject.get("result")
|
||||
d['G'] = resultMap.get("city")
|
||||
d['H'] = resultMap.get("district")
|
||||
d['I'] = resultMap.get("name")
|
||||
def type = resultMap.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
d['J'] = type
|
||||
}
|
||||
if(resultMap.get("avgPrice")){
|
||||
d['K'] = resultMap.get("avgPrice")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def loupanLevelTest(){
|
||||
ExcelUtils.modify("D:\\数据\\卓信\\上海 - 副本.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
@@ -125,6 +292,265 @@ class ApiTest {
|
||||
return dbo
|
||||
}
|
||||
|
||||
def jsAbcHouseCastTest(){
|
||||
def citys = ["无锡"]
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","estate_type2"])
|
||||
def rets = getHouseCases(districts)
|
||||
ExcelUtils.write("D:\\数据\\江苏\\房源样本.xlsx", rets, city)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def getHouseCases(districts){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
rets << ["城市","区域","名称", "物业类型", "最小案例","最大案例", "案例"]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def propertyTypeCode = null
|
||||
if(d.estate_type2=="普通住宅"){
|
||||
propertyTypeCode = "01"
|
||||
}else if(d.estate_type2=="别墅"){
|
||||
propertyTypeCode = "02"
|
||||
}else if(d.estate_type2=="商住"){
|
||||
propertyTypeCode = "03"
|
||||
}
|
||||
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCase(d.city, d.region, d.name, propertyTypeCode)
|
||||
if(results){
|
||||
def row = []
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
row << d.estate_type2
|
||||
row << results.minUnitPrice
|
||||
row << results.maxUnitPrice
|
||||
|
||||
def datas = results.datas
|
||||
if(datas){
|
||||
def str = ""
|
||||
for(def r : datas){
|
||||
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
|
||||
}
|
||||
row << str
|
||||
}
|
||||
|
||||
rets << row
|
||||
}
|
||||
println totalCount.get()+" "+d.city +" "+d.region+" "+d.name
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def arroundFacilities(){
|
||||
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city","region","name","estate_type2"])
|
||||
// districts = districts[0..1000]
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
rets << ["城市","区域","名称", "重点学校","学校", "公交","地铁","超市","医院","银行"]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def row = []
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
def facilityObj = OpenApiClient.facilitys(d.city, d.region, d.name)
|
||||
if(facilityObj && facilityObj.result){
|
||||
|
||||
def result = facilityObj.result
|
||||
if(result.primarySchool){
|
||||
row <<result.primarySchool.join(",").replace('"',"")
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.school){
|
||||
row << result.school.join(",").replace('"',"")
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.bus){
|
||||
row << result.bus.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
|
||||
if(result.metro){
|
||||
row << result.metro.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
|
||||
if(result.supermarket){
|
||||
row << result.supermarket.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.hospital){
|
||||
row << result.hospital.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.bank){
|
||||
row << result.bank.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
println totalCount.get() + " "+d.region +" "+d.name
|
||||
rets << row
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\江苏\\南京周边配套.xlsx", rets, "南京")
|
||||
|
||||
}
|
||||
|
||||
def arroundFacilitiesByAddress(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['B']){
|
||||
def facilityObj = OpenApiClient.facilitysByAddress(d['A']+d['B'])
|
||||
if(facilityObj && facilityObj.result){
|
||||
def result = facilityObj.result
|
||||
d['G'] = result.city
|
||||
d['H'] = result.region
|
||||
d['I'] = result.districtName
|
||||
if(result.primarySchool){
|
||||
d['J'] = result.primarySchool.join(",").replace('"',"")
|
||||
}else {
|
||||
d['J'] = ""
|
||||
}
|
||||
|
||||
if(result.school){
|
||||
d['K'] = result.school.join(",").replace('"',"")
|
||||
}else {
|
||||
d['K'] = ""
|
||||
}
|
||||
|
||||
if(result.bus){
|
||||
d['L'] = result.bus.join(",").replace('"',"")
|
||||
}else{
|
||||
d['L'] = ""
|
||||
}
|
||||
|
||||
|
||||
if(result.metro){
|
||||
d['M'] = result.metro.join(",").replace('"',"")
|
||||
}else{
|
||||
d['M'] = ""
|
||||
}
|
||||
|
||||
if(result.bank){
|
||||
d['N'] = result.bank.join(",").replace('"',"")
|
||||
}else{
|
||||
d['N'] = ""
|
||||
}
|
||||
|
||||
|
||||
if(result.supermarket){
|
||||
d['O'] = result.supermarket.join(",").replace('"',"")
|
||||
}else{
|
||||
d['O'] = ""
|
||||
}
|
||||
|
||||
if(result.hospital){
|
||||
d['P'] = result.hospital.join(",").replace('"',"")
|
||||
}else{
|
||||
d['P'] = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def jsAbcHouseCastTestByAddress(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['B'] && !d['H']){
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCaseByAddress(d['A']+d['B'], d['C'])
|
||||
if(results){
|
||||
d['H'] = results.minUnitPrice
|
||||
d['I'] = results.maxUnitPrice
|
||||
|
||||
def datas = results.datas
|
||||
if(datas){
|
||||
def str = ""
|
||||
for(def r : datas){
|
||||
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
|
||||
}
|
||||
d['J'] = str
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def houseValueInformation(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.houseValueInformation(d['A'], d['B'], d['C'])
|
||||
if(results) {
|
||||
def addressValue = results.addressValue
|
||||
if (addressValue) {
|
||||
d['D'] = addressValue.city + "-" + addressValue.district + "-" + addressValue.name + "-" + addressValue.avgPrice + "-" + addressValue.totalPrice
|
||||
}
|
||||
def communityAvgPrice = results.communityAvgPrice
|
||||
if (communityAvgPrice) {
|
||||
d['E'] = communityAvgPrice.toString()
|
||||
}
|
||||
def surroundingPoi = results.surroundingPoi
|
||||
if (surroundingPoi) {
|
||||
d['F'] = surroundingPoi.toString()
|
||||
}
|
||||
def house = results.houseCase
|
||||
if (house) {
|
||||
d['G'] = house.minUnitPrice
|
||||
d['H'] = house.maxUnitPrice
|
||||
|
||||
def datas = house.datas
|
||||
if (datas) {
|
||||
def str = ""
|
||||
for (def r : datas) {
|
||||
str = str + r.buildingName + "-" + r.strctureArea + "-" + r.unitPrice + "-" + r.sampleDate.substring(0, 10) + "\n"
|
||||
}
|
||||
d['J'] = str
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+73
-11
@@ -39,8 +39,10 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
|
||||
|
||||
//
|
||||
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
|
||||
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
|
||||
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
|
||||
|
||||
|
||||
//"alias":"alias",
|
||||
//ownership_type:权属类型 为商品房、经适房、动迁房等
|
||||
@@ -51,6 +53,9 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
|
||||
"loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
|
||||
|
||||
|
||||
// def fieldsMap = ["household_count":"households"]
|
||||
|
||||
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
|
||||
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
|
||||
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
|
||||
@@ -67,7 +72,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
|
||||
def analysRets = []
|
||||
|
||||
@@ -88,12 +93,12 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
||||
if(totalCount){
|
||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)
|
||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
|
||||
}
|
||||
}
|
||||
println analysRets.toString()
|
||||
sendStatEmail(headers, analysRets, rh_analysRets)
|
||||
// sendStatEmail(headers, analysRets, rh_analysRets)
|
||||
// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
|
||||
}
|
||||
|
||||
@@ -106,11 +111,11 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def matchRets2 = selectRestByCredit(matchRets1)
|
||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2)
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
|
||||
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
//updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
@@ -199,11 +204,13 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def processFieldsFromMatch(c,matchRets){
|
||||
def processFieldsFromMatch(c,matchRets, esWebSource){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
fields << "city"
|
||||
fields << "estate_type2"
|
||||
def fieldValueMap = [:]
|
||||
def rets = []
|
||||
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
|
||||
for(def matchDist : matchRets){
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
@@ -212,11 +219,13 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
def newValues = strandField(c, dist, matchDist)
|
||||
if(newValues){
|
||||
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
|
||||
println dist._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
}
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource)
|
||||
return fieldValueMap;
|
||||
}
|
||||
|
||||
@@ -332,7 +341,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
def findEsDatas(c, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' ", {r->
|
||||
list << r
|
||||
})
|
||||
return list
|
||||
@@ -379,6 +388,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = matching(data)
|
||||
// def ret = newMatching(data)
|
||||
if(ret){
|
||||
// println totalCount.get() +" "+data.city+" "+data.name+" [Match] "+data.match_region+" "+data.match_name +" "+data.flag+" "+data.credit
|
||||
rets << ret
|
||||
@@ -399,7 +409,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def selectRestByCredit(matchRets){
|
||||
def matchRets1 = []
|
||||
for(def r : matchRets){
|
||||
if(r.credit && r.credit>=0.9){
|
||||
if(r.credit && r.credit>0.9){
|
||||
matchRets1 << r
|
||||
}
|
||||
}
|
||||
@@ -458,6 +468,58 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def newMatching(data){
|
||||
def name = data.name
|
||||
if(data.name && data.name.contains("·")){
|
||||
name = data.name.replace("·", "")
|
||||
}
|
||||
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(data.city, data.region, name)
|
||||
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district")
|
||||
flag = "映射别名匹配"
|
||||
}
|
||||
if(!dist) {
|
||||
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district")
|
||||
if(rets && rets.size()==1){
|
||||
flag = "别名匹配"
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
|
||||
if(dist){
|
||||
data.flag = flag
|
||||
data.match_id = dist._id.toString()
|
||||
data.match_region = dist.region
|
||||
data.match_name = dist.name
|
||||
data.credit = 0.95
|
||||
}else{
|
||||
def jsonObject = FangjiaClientUtils.newMatch(data.city, data.region+name, "property", false)
|
||||
if(jsonObject!=null && jsonObject.credit>0){
|
||||
if(data.address && isStandardRoad(data.address)){
|
||||
jsonObject = FangjiaClientUtils.newMatch(data.city, data.address, "property", false)
|
||||
}else{
|
||||
data.flag = "匹配接口-地址"
|
||||
}
|
||||
}else{
|
||||
data.flag = "匹配接口-名称"
|
||||
}
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
data.match_id = jsonObject.get("id")
|
||||
data.match_region = jsonObject.get("district")
|
||||
data.match_name = jsonObject.get("name")
|
||||
data.credit = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
if(data.match_id){
|
||||
return data
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def updateDB(c, updateRets){
|
||||
|
||||
for(def rm : updateRets) {
|
||||
|
||||
+196
@@ -114,6 +114,27 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static def structure(def city, def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
if(address){
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def arroundPrice(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
@@ -172,4 +193,179 @@ class FangjiaClientUtils {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1"
|
||||
if(floor){
|
||||
url = url + "&floor="+floor.toString().toDouble().toInteger()
|
||||
}
|
||||
if(maxFloor){
|
||||
url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger()
|
||||
}
|
||||
if(room){
|
||||
url = url + "&room="+room.toString().toDouble().toInteger()
|
||||
}
|
||||
if(hall){
|
||||
url = url + "&hall="+hall.toString().toDouble().toInteger()
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static def newMatch(city,address, property, useMap){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
def url = null;
|
||||
try {
|
||||
address = address.replace("{","").replace("\\","")
|
||||
url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
|
||||
if(city){
|
||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
if(property){
|
||||
url = url + "&category=${property}"
|
||||
}
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println "±¨´íµØÖ·£º"+url
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def newMatchPrice(city,address, property, useMap){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://open.fangjia.com/addrSvc/avgPrice?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
|
||||
if(city){
|
||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
if(property){
|
||||
url = url + "&category=${property}"
|
||||
}
|
||||
url = url + "&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static def newStructureByLevel9(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","").replace("\\","")
|
||||
def url = "http://192.168.0.100:32080/FangJiaAddressService/v2/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
|
||||
if(city){
|
||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static def newStandard(address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def formatFivelevel(city, address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static def formatNinelevel(city, address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static def villageIdentify(city, address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify?address="+URLEncoder.encode(address,"UTF-8")+"&type=2"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+33
-4
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
// strandField(offices)
|
||||
// }
|
||||
|
||||
|
||||
|
||||
// addBlock()
|
||||
|
||||
// addFieldValue()
|
||||
|
||||
// updateErrorField()
|
||||
export()
|
||||
test()
|
||||
// export()
|
||||
// test()
|
||||
// addOffice()
|
||||
fileAddBlock()
|
||||
}
|
||||
|
||||
def export(){
|
||||
@@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
|
||||
def fileAddBlock(){
|
||||
|
||||
def rets = findAllBlockRange()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d->
|
||||
def location = d['T']
|
||||
if(location && !d['F']){
|
||||
def arrs = location.split(",")
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
|
||||
if(baidu){
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
|
||||
def lastObj = null
|
||||
for (def r : rets) {
|
||||
def flag = InsidePolygon.isInArea(lng2, lat2, r.points)
|
||||
if (flag && r.name) {
|
||||
lastObj = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lastObj) {
|
||||
d['F'] = lastObj.name
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
def city = "常州"
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"])
|
||||
println districts.size()
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称", "房价网ToC", "房价网ToB", "近三个月成交均价"]
|
||||
def index = 0
|
||||
for(def d : districts){
|
||||
index++
|
||||
def ret = []
|
||||
def dbo = CommonDatautils.queryDistrictPrice(d.city, d.region, d.name, DateUtils.parseFromTightByPattern("2023-09-17","yyyy-MM-dd"))
|
||||
ret << d.city
|
||||
ret << d.region
|
||||
ret << d.name
|
||||
if(dbo){
|
||||
ret << dbo.avg_price
|
||||
ret << dbo.avg_price_2b
|
||||
}
|
||||
|
||||
|
||||
def map = CcbCommonUtils.findTrades(d, "2023-05-01","2023-09-01")
|
||||
if(map){
|
||||
ret << map.avg_price
|
||||
}
|
||||
rets << ret
|
||||
println index
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\江南农商行\\常州小区价格.xlsx", rets, "数据")
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.fangjia.script.gj.districtwash
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class DistrictTypeUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictTypeUpdate();
|
||||
// task.run()
|
||||
task.updateType()
|
||||
}
|
||||
|
||||
def updateType(){
|
||||
ExcelUtils.modify("D:\\数据\\厦门\\物业类型可疑小区(第二批).xlsx",{d->
|
||||
def flag = d['F']
|
||||
if(flag && flag.toString().toDouble().toInteger()==2){
|
||||
def name = d['C']
|
||||
def type = null
|
||||
if(name && name.toString().contains("商住")){
|
||||
type = "商住"
|
||||
}else{
|
||||
type = "别墅"
|
||||
}
|
||||
|
||||
|
||||
def udata = [:]
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
udata.put("name", d['C'])
|
||||
udata.put("estateType2", type)
|
||||
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println d['C'] +" "+s
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def datas = []
|
||||
citys.each{c->
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name","estate_type2", "c_date"])
|
||||
districts.each{d->
|
||||
def row = null
|
||||
|
||||
if(d.name.contains("商住")){
|
||||
if(d.estate_type2!="商住") {
|
||||
row = [d.city, d.region, d.name, d.estate_type2, DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd")]
|
||||
}
|
||||
}else{
|
||||
if(d.name.contains("墅")){
|
||||
if(d.estate_type2!="别墅"){
|
||||
row = [d.city, d.region, d.name, d.estate_type2, DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd")]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(row){
|
||||
datas << row
|
||||
println row
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\小区\\物业类型可疑小区.xlsx", datas, "小区")
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
def datas = findHouseCount("北京", 20230201, 20230430)
|
||||
ExcelUtils.write("D:\\数据\\天弘基金.xlsx", datas, "样例")
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def startDate, def endDate){
|
||||
def rets = []
|
||||
try{
|
||||
// String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='ke.com' order by p_date desc"
|
||||
def startMdate = DateUtils.parseFromTightByPattern(startDate.toString(),"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
|
||||
rets << ["城市","区域","板块","名称","面积","单价","总价","室","厅","卫","装修","朝向","发布时间","物业类型","楼层状况","楼高","建筑结构","建筑年代","标题","URL"]
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
// QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(QueryBuilders.termQuery("root", "ke.com"))
|
||||
.must(date_range).must(area_range)//.must(date_range_mdate)
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses) {
|
||||
|
||||
for (int k = 0; k < houses.size(); k++) {
|
||||
println k
|
||||
def data = houses[k];
|
||||
def r = []
|
||||
r << data.city
|
||||
r << data.region
|
||||
r << data.block
|
||||
r << data.district_name
|
||||
r << data.area
|
||||
r << data.avg_price
|
||||
r << data.total
|
||||
r << data.room
|
||||
r << data.hall
|
||||
r << data.toilet
|
||||
r << data.fitment
|
||||
r << data.direction
|
||||
r << data.p_date
|
||||
r << data.estate_type
|
||||
r << data.floor_info
|
||||
r << data.height
|
||||
r << data.build_structure
|
||||
r << data.build_date
|
||||
r << data.name
|
||||
r << data.url
|
||||
rets << r
|
||||
if(rets.size()>500){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return rets
|
||||
}
|
||||
+2
-2
@@ -25,8 +25,8 @@ class DistrictLocationBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationBatch()
|
||||
// task.run()
|
||||
task.deleteDist()
|
||||
task.run()
|
||||
// task.deleteDist()
|
||||
}
|
||||
|
||||
def deleteDist(){
|
||||
|
||||
+4
-1
@@ -28,7 +28,10 @@ class GaoDeMapSearchTest {
|
||||
// task.mapSearch(path)
|
||||
// task.baiduMapSearch(path)
|
||||
// task.checkTrue(path)
|
||||
task.addFinancialLevel(path)
|
||||
// task.addFinancialLevel(path)
|
||||
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.financialRating("重庆", "凤中路193号")
|
||||
println results
|
||||
}
|
||||
|
||||
def addFinancialLevel(path){
|
||||
|
||||
+289
@@ -0,0 +1,289 @@
|
||||
package com.fangjia.script.gj.meishubao
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
class MsbPocTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new MsbPocTest();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
// matchOld(inputfile)
|
||||
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
|
||||
// test("D:\\数据\\美术宝\\test2.xlsx")
|
||||
|
||||
// cityPrice()
|
||||
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
|
||||
|
||||
addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx")
|
||||
}
|
||||
|
||||
def cityPrice(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\临时\\城市房产价格.xlsx",0,1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
map.put(d['A'], d)
|
||||
}
|
||||
|
||||
def ccbdatas = ExcelUtils.read("D:\\数据\\临时\\建行城市均价.xlsx",0,1)
|
||||
def ccbmap = [:]
|
||||
for(def d : ccbdatas){
|
||||
ccbmap.put(d['A'], d['B'])
|
||||
}
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\临时\\房价网城市均价.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
def d = map.get(row['A'])
|
||||
if(!d){
|
||||
d = map.get(row['A'].replace('州',''))
|
||||
}
|
||||
if(d){
|
||||
row['C'] = d[1]
|
||||
row['D'] = d[2]
|
||||
}
|
||||
|
||||
|
||||
def d1 = ccbmap.get(row['A'])
|
||||
if(!d1){
|
||||
d1 = ccbmap.get(row['A'].replace('州',''))
|
||||
}
|
||||
if(d1){
|
||||
row['E'] = d1
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matchOld(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['S'] && row['D']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def name = row['F']
|
||||
def address = row['H']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['S'] = jsonObject.get("id")
|
||||
row['T'] = jsonObject.get("city")
|
||||
row['U'] = jsonObject.get("district")
|
||||
row['V'] = jsonObject.get("name")
|
||||
row['W'] = jsonObject.get("propertyType")
|
||||
row['X'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def test(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
def jsonObject = FangjiaClientUtils.newMatch(null, row['A'], null, true)
|
||||
println row['A']
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
row['B'] = jsonObject.get("id")
|
||||
row['C'] = jsonObject.get("city")
|
||||
row['D'] = jsonObject.get("district")
|
||||
row['E'] = jsonObject.get("name")
|
||||
def type = jsonObject.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['F'] = type
|
||||
}
|
||||
row['G'] = jsonObject.get("credit")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def match(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
row['J'] = jsonObject.get("id")
|
||||
row['K'] = jsonObject.get("city")
|
||||
row['L'] = jsonObject.get("district")
|
||||
row['M'] = jsonObject.get("name")
|
||||
def type = jsonObject.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['N'] = type
|
||||
}
|
||||
row['O'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def mergeMatchPrice(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['L']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def name = row['F']
|
||||
def address = row['G']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, "property", false)
|
||||
if(jsonObject && !jsonObject.containsKey("result")){
|
||||
jsonObject = FangjiaClientUtils.newMatchPrice(city, address, "property", false)
|
||||
}
|
||||
if(jsonObject && !jsonObject.containsKey("result")){
|
||||
jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, null, true)
|
||||
}
|
||||
|
||||
if(jsonObject && !jsonObject.containsKey("result")){
|
||||
jsonObject = FangjiaClientUtils.newMatchPrice(city, address, null, true)
|
||||
}
|
||||
if(jsonObject && jsonObject.containsKey("result")) {
|
||||
def resultMap = jsonObject.get("result")
|
||||
row['L'] = resultMap.get("city")
|
||||
row['M'] = resultMap.get("district")
|
||||
row['N'] = resultMap.get("name")
|
||||
def type = resultMap.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['O'] = type
|
||||
}
|
||||
if(resultMap.get("avgPrice")){
|
||||
row['P'] = resultMap.get("avgPrice")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addCCBPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['M']){
|
||||
def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
if(dist){
|
||||
def d = CommonDatautils.findDistrictById(dist._id.toString())
|
||||
if(d){
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
|
||||
if(map){
|
||||
row['M'] = map.avg_price
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['I'] && !row['P']){
|
||||
row['I'] = row['I'].toString().trim()
|
||||
def d = CommonDatautils.findDistrictById(row['I'])
|
||||
if(d){
|
||||
row['K'] = d.city
|
||||
row['L'] = d.region
|
||||
row['M'] = d.name
|
||||
row['T'] = d.estate_type2
|
||||
row['U'] = d.cat
|
||||
row['V'] = d.status
|
||||
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice(d.city, d.region, d.name, 202303,202310)
|
||||
def priceMap = cityMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['N'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("beike_esf_price")){
|
||||
row['N'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("anjuke_esf_price")){
|
||||
row['O'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
|
||||
if(dbo){
|
||||
row['P'] = dbo.avg_price
|
||||
}
|
||||
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(row['I'], cityId)
|
||||
if(map){
|
||||
row['Q'] = map.avg_price
|
||||
}
|
||||
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-10-01')
|
||||
if(tradePrice){
|
||||
row['R'] = tradePrice.toInteger()
|
||||
}
|
||||
|
||||
def gxdPrice = CommonDatautils.findGXDPrice(d.city, d.region, d.name)
|
||||
d['S'] = gxdPrice
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def standard(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(row['G']){
|
||||
|
||||
row['G'] = row['F'].replace('·','').replace("小区","").replace("花园","").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
// println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
// def jsonObject = FangjiaClientUtils.newStandard(row['G'])
|
||||
// if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// row['Q'] = jsonObject.get("name")
|
||||
// row['R'] = jsonObject.get("id")
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice(row['D'], row['E'], row['F'])
|
||||
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['L'] = priceMap.get("lianjia_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("beike_esf_price")){
|
||||
row['M'] = priceMap.get("beike_esf_price")
|
||||
}
|
||||
|
||||
if(priceMap.containsKey("anjuke_esf_price")){
|
||||
row['N'] = priceMap.get("anjuke_esf_price")
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
+51
-23
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
WebDriver driver = null;
|
||||
|
||||
//广州,佛山 重庆
|
||||
static main(args) {
|
||||
@@ -50,7 +50,7 @@ public class BeiKeCrawlByChrome {
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver = new ChromeDriver()
|
||||
def citys = getAllCityUrls(driver)
|
||||
|
||||
driver = getLogin(driver, "https://sh.ke.com/")
|
||||
@@ -93,19 +93,28 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
try{
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
println "异常Url:"+url
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
return driver;
|
||||
}
|
||||
|
||||
def crawlData(driver, citys){
|
||||
@@ -130,7 +139,8 @@ public class BeiKeCrawlByChrome {
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
driver = checkDocument(driver, newUrl);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -142,8 +152,9 @@ public class BeiKeCrawlByChrome {
|
||||
regionText = "昆山"
|
||||
}
|
||||
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println url +" 详情链接:"+detailUrls.size()
|
||||
def map = getPageDeail(driver, newCity, href, regionText)
|
||||
driver = map.driver
|
||||
println url +" 详情链接:"+map.detailUrls.size()
|
||||
}
|
||||
}
|
||||
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
|
||||
@@ -191,9 +202,9 @@ public class BeiKeCrawlByChrome {
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
// def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = checkDocument(driver, url);
|
||||
driver = checkDocument(driver, url);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
@@ -203,10 +214,11 @@ public class BeiKeCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
println "开始查库 "+new Date().toString()
|
||||
def dbo = findTrade(durl , "贝壳找房")
|
||||
|
||||
println "结束查库"+new Date().toString()
|
||||
if (!dbo) {
|
||||
parse(city, e, region, driver, null)
|
||||
driver = parse(city, e, region, driver, null)
|
||||
}
|
||||
// else if(dbo && !dbo.lift){
|
||||
// parse(city, e, region, driver, dbo._id)
|
||||
@@ -225,7 +237,10 @@ public class BeiKeCrawlByChrome {
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
def m = [:]
|
||||
m.driver = driver
|
||||
m.detailUrls = detailUrls
|
||||
return m
|
||||
}
|
||||
|
||||
def parse(city,e, region, driver, id){
|
||||
@@ -233,8 +248,10 @@ public class BeiKeCrawlByChrome {
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
driver = checkDocument(driver, detailUrl);
|
||||
def detailDoc = Jsoup.parse(driver.getPageSource())
|
||||
|
||||
println "打开获取页面 "+new Date().toString()
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -335,18 +352,21 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println "解析页面完成 "+new Date().toString()
|
||||
println detail
|
||||
if(id){
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
|
||||
}else{
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
println "保存修改完成 "+new Date().toString()
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
return driver
|
||||
}
|
||||
|
||||
|
||||
@@ -469,16 +489,16 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = "https:"+e.attr("href")
|
||||
// if (citys.contains(e.text())){
|
||||
if (citys.contains(e.text())){
|
||||
list << [e.text(), href]
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver, url){
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
|
||||
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
@@ -494,17 +514,25 @@ public class BeiKeCrawlByChrome {
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("17148854005");
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
// elemUsername.sendKeys("16283166371");
|
||||
// elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
|
||||
return driver
|
||||
}
|
||||
|
||||
|
||||
+61
-38
@@ -34,6 +34,7 @@ public class LianjiaCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
WebDriver driver = null;
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -42,6 +43,7 @@ public class LianjiaCrawlByChrome {
|
||||
// task.addDealRegion();
|
||||
// task.standardRegion()
|
||||
// task.updadteDealPriceField()
|
||||
|
||||
}
|
||||
|
||||
def updadteDealPriceField(){
|
||||
@@ -64,7 +66,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run1(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
@@ -122,7 +124,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
// cityMap = cityMap.reverse()
|
||||
driver = getLogin(driver, "https://sh.lianjia.com/")
|
||||
@@ -206,17 +208,11 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
def cityUrl = ret[1]
|
||||
def newUrl = cityUrl+"chengjiao/"
|
||||
try{
|
||||
driver.get(newUrl)
|
||||
}catch(Exception e){
|
||||
println newUrl +" 链接有问题!"
|
||||
continue
|
||||
}
|
||||
|
||||
// driver.get(newUrl)
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
|
||||
// Thread.sleep(10000)
|
||||
driver = checkDocument(driver, newUrl);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -226,8 +222,9 @@ public class LianjiaCrawlByChrome {
|
||||
regionText = "昆山"
|
||||
}
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
def map = getPageDeail(driver, newCity, href, regionText)
|
||||
driver = map.driver
|
||||
println newUrl +" 详情链接:"+map.detailUrls.size()
|
||||
}
|
||||
}
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
@@ -267,30 +264,42 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
try{
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
return driver;
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl, regionText){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
// driver.get(url)
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def doc = checkDocument(driver, url);
|
||||
driver = checkDocument(driver, url);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
@@ -303,7 +312,7 @@ public class LianjiaCrawlByChrome {
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver, null)
|
||||
driver = parse(city, e, regionText, driver, null)
|
||||
}
|
||||
// else if(dbo && !dbo.lift){
|
||||
// parse(city, e, regionText, driver, dbo._id)
|
||||
@@ -326,7 +335,10 @@ public class LianjiaCrawlByChrome {
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
def m = [:]
|
||||
m.driver = driver
|
||||
m.detailUrls = detailUrls
|
||||
return m
|
||||
}
|
||||
|
||||
def parse(city,e, regionText, driver, id){
|
||||
@@ -334,8 +346,11 @@ public class LianjiaCrawlByChrome {
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if(price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
|
||||
// driver.get(detailUrl)
|
||||
driver = checkDocument(driver, detailUrl);
|
||||
def detailDoc = Jsoup.parse(driver.getPageSource())
|
||||
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -450,6 +465,7 @@ public class LianjiaCrawlByChrome {
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
return driver;
|
||||
}
|
||||
|
||||
|
||||
@@ -571,7 +587,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityList << [e.text(), href]
|
||||
}
|
||||
}
|
||||
@@ -593,16 +609,23 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
elemUsername.sendKeys("17148854005");
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
// elemUsername.sendKeys("17740818975");
|
||||
// elemPassword.sendKeys("13qeadzc");
|
||||
|
||||
// elemUsername.sendKeys("16283166371");
|
||||
// elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(15000)
|
||||
Thread.sleep(20000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(10000)
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
return driver
|
||||
}
|
||||
|
||||
+39
-1
@@ -3,7 +3,10 @@ package com.fangjia.script.gj.paimai
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -35,7 +38,8 @@ class PaimaiTest {
|
||||
// task.updateFieldPaimai()
|
||||
// task.statCity()
|
||||
// task.jsStat()
|
||||
task.selectData()
|
||||
// task.selectData()
|
||||
task.randomExtractData()
|
||||
}
|
||||
|
||||
def selectData(){
|
||||
@@ -768,4 +772,38 @@ class PaimaiTest {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def randomExtractData(){
|
||||
def runedcity = ["广州","重庆","南京","佛山","东莞","扬州","镇江","南通","惠州","珠海","连云港","淮安","清远","云浮","上海","深圳","苏州","徐州","盐城","宿迁","肇庆","韶关","汕尾","常州","湛江","泰州","无锡","江门","汕头","中山","合肥","河源","阳江","梅州","茂名","揭阳","福州","潮州"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def datas = []
|
||||
datas << ["城市", "标地物", "面积", "URL", "结束时间", "成交价"]
|
||||
for(def c : citys){
|
||||
if(runedcity.contains(c)){
|
||||
continue
|
||||
}
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", c)
|
||||
q.put("status", "已成交")
|
||||
q.put("estateType","住宅")
|
||||
q.put("area", new BasicDBObject('$exists', true))
|
||||
q.put("title",new BasicDBObject('$exists', true) )
|
||||
def dbcursor = paimai43_COLL.find(q).sort(new BasicDBObject("finishDate",-1)).limit(100)
|
||||
def rows = []
|
||||
for(def d : dbcursor){
|
||||
if(d.title && !d.title.contains("商") && !d.title.contains("铺") && !d.title.contains("车位")){
|
||||
def date = DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd")
|
||||
rows << [d.city, d.title, d.area, d.url, date, d.tradePrice]
|
||||
}
|
||||
if(rows.size()>=40){
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
datas.addAll(rows)
|
||||
println c +" "+rows.size()
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\北交所\\拍卖案例-住宅-测试.xlsx", datas, "数据1")
|
||||
}
|
||||
}
|
||||
|
||||
+122
-1
@@ -9,8 +9,19 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 价格对比,第三方,成交,建行价格,自已家价格
|
||||
*/
|
||||
class AddPriceTest {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
@@ -26,10 +37,120 @@ class AddPriceTest {
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
addFjwPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
exportPkPrice()
|
||||
// addRegion()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
d['C'] = dist.region
|
||||
})
|
||||
}
|
||||
def exportPkPrice(){
|
||||
//"南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安",
|
||||
def citys = ["宿迁"]
|
||||
|
||||
for(def c : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","alias"])
|
||||
println c+" district:"+districts.size()
|
||||
|
||||
def fjwPriceMap =[:]// CommonDatautils.queryAllDistrictPrice(c, DateUtils.parseFromTightByPattern("2023-10-08","yyyy-MM-dd"))
|
||||
def datas = addAllPrice(districts, fjwPriceMap)
|
||||
ExcelUtils.write("D:\\数据\\上海农总行\\江苏省价格比对.xlsx", datas, c)
|
||||
println c +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def addAllPrice(districts, fjwPriceMap){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
rets << ["id", "城市","区域","名称","建行价格","计算方式","可信度","房价网ToB价格","计算方式","可信度","近6个月成交均价"]
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createPriceData(cityCodeDataMap, dist, fjwPriceMap)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createPriceData(cityCodeDataMap, d, fjwPriceMap){
|
||||
def row = []
|
||||
row << d._id.toString()
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = findAvgPrice(d._id.toString(), cityId)
|
||||
if(map){
|
||||
row << map.avg_price
|
||||
row << map.calc_type
|
||||
row << map.credit_value
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
}
|
||||
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
|
||||
// def dbo = fjwPriceMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
if(dbo){
|
||||
row << dbo.avg_price
|
||||
if(dbo.value_type == 1){
|
||||
row << "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
row << "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
row << "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
row << "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
row << "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
row << "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
row << "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
row << "无方式"
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << dbo.credit
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
}
|
||||
|
||||
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-09-01')
|
||||
if(tradePrice){
|
||||
row << tradePrice.toInteger()
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
|
||||
//def json = GaoDeApiUtils.geo("上海市", "上海市浦东新区永泰路1138弄11号")
|
||||
//def locationStr = null
|
||||
//if(json && json.count>=1){
|
||||
// def m = json.geocodes[0]
|
||||
// if(m){
|
||||
// locationStr = m.location
|
||||
// }
|
||||
//}
|
||||
|
||||
def location = ["lng":121.526016,"lat":31.143047]
|
||||
|
||||
|
||||
//def rows = []
|
||||
//rows << ["城市","区域","名称","地址","距离","均价"]
|
||||
//if(location && location.lat){
|
||||
//
|
||||
// def dbos = CommonDatautils.getDistrictArround("上海", 121.526016, 31.143047)
|
||||
// for(def d : dbos){
|
||||
//
|
||||
// def dbo = CommonDatautils.findDistrictPrice(d.city, d.region, d.name);
|
||||
// def avg_price = null
|
||||
// if(dbo){
|
||||
// avg_price = dbo.avg_price
|
||||
// }
|
||||
//
|
||||
// def distance = GeoUtils.calculateDistance(location.lng, location.lat, d.lng/10000000000.0d, d.lat/10000000000.0)
|
||||
//
|
||||
// rows << [d.city, d.region, d.name, d.address, distance.intValue(), avg_price, d.status]
|
||||
// }
|
||||
//
|
||||
//// def map = CommonDatautils.findMaxLatLngMinLatLng(location.lat, location.lng, 1000);
|
||||
//// def ret = CommonDatautils.findInRangeDist("上海", "district", (map.maxLat * 10000000000d).toLong(), (map.minLat * 10000000000d).toLong(), (map.maxLng * 10000000000d).toLong(), (map.minLng * 10000000000d).toLong());
|
||||
////
|
||||
//// if(ret){
|
||||
//// for(def row : ret){
|
||||
//// if(row.lat2 && row.lng2){
|
||||
//// def lat2 = row.lat2/10000000000.0;
|
||||
//// def lng2 = row.lng2/10000000000.0;
|
||||
//// region = row.region;
|
||||
//// def distance = GeoHelper.GetDistance(location.lat, location.lng, lat2, lng2)
|
||||
//// if(distance <= 1000){
|
||||
//// def dbo = CommonDatautils.findDistrictPrice(row.city, row.region, row.name);
|
||||
//// def avg_price = null
|
||||
//// if(dbo){
|
||||
//// avg_price = dbo.avg_price
|
||||
//// }
|
||||
//// rows << [row.city, row.region, row.name, row.address, distance.intValue(), avg_price]
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
// ExcelUtils.write("D:\\数据\\临时\\上海市浦东新区永泰路1138弄11号.xlsx", rows, "一公里楼盘")
|
||||
//}
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{d->
|
||||
def loc = d['AB']
|
||||
if(loc){
|
||||
def arrs = loc.split(",")
|
||||
def distance = GeoUtils.calculateDistance(location.lng, location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
|
||||
d['AD'] = distance
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
|
||||
|
||||
//def distance = GeoUtils.calculateDistance(121.526016, 31.143047, 111.202375, 35.592230)
|
||||
//println distance
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
package com.fangjia.script.gj.township
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class StreetProcess {
|
||||
|
||||
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
|
||||
|
||||
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
|
||||
|
||||
|
||||
final DBCollection POLYLINE235 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new StreetProcess();
|
||||
// println RegionData.formatRegion("鞍山","铁东区")
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
//
|
||||
// def region = RegionData.formatRegion("三明市", "三元区")
|
||||
// println region
|
||||
|
||||
// standard()
|
||||
// matchStreet()
|
||||
synchTownship()
|
||||
}
|
||||
|
||||
def synchTownship(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city_name","福州市")
|
||||
// q.put("shape",new BasicDBObject('$exists', true))
|
||||
def dbcursor = TOWNSHIP.find(q)
|
||||
for(def d : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
// udata.put("shape", d.shape)
|
||||
udata.put("area_station", d.area_station)
|
||||
TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
def matchStreet(){
|
||||
def polylineMap = findStreetPolyline()
|
||||
def q = new BasicDBObject()
|
||||
// q.put("fangjia_city", "福州")
|
||||
// q.put("shape",new BasicDBObject('$exists', false))
|
||||
def dbcursor = TOWNSHIP.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def streetMap = polylineMap.get(d.fangjia_city+"##"+d.fangjia_area)
|
||||
if(streetMap){
|
||||
def alias = d.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(d.street_name)){
|
||||
alias << d.street_name
|
||||
}
|
||||
for(def streetName : alias){
|
||||
def polyline = streetMap.get(streetName)
|
||||
if(polyline){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
TOWNSHIP.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
println index + " "+ d.get("_id")
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findStreetPolyline(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("fjw_city","福州")
|
||||
def dbcursor = POLYLINE235.find(q)
|
||||
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.fjw_city+"##"+d.fjw_region
|
||||
def streetMap = map.get(key)
|
||||
if(!streetMap){
|
||||
streetMap = [:]
|
||||
map.put(key, streetMap)
|
||||
}
|
||||
streetMap.put(d.town, d.polyline)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def standard(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fjw_city",new BasicDBObject('$exists', false))
|
||||
def dbcursor = POLYLINE235.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index ++
|
||||
def city = RegionData.formatCity(d.city)
|
||||
def region = RegionData.formatRegion(d.city, d.region)
|
||||
// def street = null
|
||||
// if(city && region){
|
||||
// street = RegionData.formatStreet(city, region, d.town)
|
||||
// }
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fjw_city", city)
|
||||
udata.put("fjw_region", region)
|
||||
// if(street){
|
||||
// udata.put("fjw_street", street)
|
||||
// }
|
||||
POLYLINE235.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def standard1(){
|
||||
def list = TOWNSHIP.find(new BasicDBObject()).toList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(list.size())
|
||||
for(def i=0;i<list.size();i++){
|
||||
final def d = list[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def city = RegionData.formatCity(d.city_name)
|
||||
if(city){
|
||||
def fangjia_province = ProvinceData.getProvince(city)
|
||||
def region = RegionData.formatRegion(city, d.area_name)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fangjia_city", city)
|
||||
udata.put("fangjia_area", region)
|
||||
udata.put("fangjia_province", fangjia_province)
|
||||
TOWNSHIP.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println totalCount.get()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package com.fangjia.script.gj.village
|
||||
import java.util.Arrays;
|
||||
class BaiduCoordinateConverter {
|
||||
|
||||
static double[] array1 = [75, 60, 45, 30, 15, 0]
|
||||
static double[][] array2 = [
|
||||
[-0.0015702102444, 111320.7020616939, 1704480524535203.0, -10338987376042340.0, 26112667856603880.0, -35149669176653700.0, 26595700718403920.0, -10725012454188240.0, 1800819912950474.0, 82.5],
|
||||
[0.0008277824516172526, 111320.7020463578, 647795574.6671607, -4082003173.641316, 10774905663.51142, -15171875531.51559, 12053065338.62167, -5124939663.577472, 913311935.9512032, 67.5],
|
||||
[0.00337398766765, 111320.7020202162, 4481351.045890365, -23393751.19931662, 79682215.47186455, -115964993.2797253, 97236711.15602145, -43661946.33752821, 8477230.501135234, 52.5],
|
||||
[0.00220636496208, 111320.7020209128, 51751.86112841131, 3796837.749470245, 992013.7397791013, -1221952.21711287, 1340652.697009075, -620943.6990984312, 144416.9293806241, 37.5],
|
||||
[-0.0003441963504368392, 111320.7020576856, 278.2353980772752, 2485758.690035394, 6070.750963243378, 54821.18345352118, 9540.606633304236, -2710.55326746645, 1405.483844121726, 22.5],
|
||||
[-0.0003218135878613132, 111320.7020701615, 0.00369383431289, 823725.6402795718, 0.46104986909093, 2351.343141331292, 1.58060784298199, 8.77738589078284, 0.37238884252424, 7.45]
|
||||
]
|
||||
static double[] array3 = [12890594.86, 8362377.87, 5591021, 3481989.83, 1678043.12, 0];
|
||||
static double[][] array4 = [
|
||||
[1.410526172116255e-8, 0.00000898305509648872, -1.9939833816331, 200.9824383106796, -187.2403703815547, 91.6087516669843, -23.38765649603339, 2.57121317296198, -0.03801003308653, 17337981.2],
|
||||
[-7.435856389565537e-9, 0.000008983055097726239, -0.78625201886289, 96.32687599759846, -1.85204757529826, -59.36935905485877, 47.40033549296737, -16.50741931063887, 2.28786674699375, 10260144.86],
|
||||
[-3.030883460898826e-8, 0.00000898305509983578, 0.30071316287616, 59.74293618442277, 7.357984074871, -25.38371002664745, 13.45380521110908, -3.29883767235584, 0.32710905363475, 6856817.37],
|
||||
[-1.981981304930552e-8, 0.000008983055099779535, 0.03278182852591, 40.31678527705744, 0.65659298677277, -4.44255534477492, 0.85341911805263, 0.12923347998204, -0.04625736007561, 4482777.06],
|
||||
[3.09191371068437e-9, 0.000008983055096812155, 0.00006995724062, 23.10934304144901, -0.00023663490511, -0.6321817810242, -0.00663494467273, 0.03430082397953, -0.00466043876332, 2555164.4],
|
||||
[2.890871144776878e-9, 0.000008983055095805407, -3.068298e-8, 7.47137025468032, -0.00000353937994, -0.02145144861037, -0.00001234426596, 0.00010322952773, -0.00000323890364, 826088.5]
|
||||
];
|
||||
|
||||
public static double[] convertor(double x, double y, double[] param) {
|
||||
double t = param[0] + param[1] * Math.abs(x);
|
||||
double cc = Math.abs(y) / param[9];
|
||||
double cf = param[2] + param[3] * cc + param[4] * cc * cc + param[5] * cc * cc * cc + param[6] * cc * cc * cc * cc + param[7] * cc * cc * cc * cc * cc + param[8] * cc * cc * cc * cc * cc * cc;
|
||||
t *= (x < 0) ? -1 : 1;
|
||||
cf *= (y < 0) ? -1 : 1;
|
||||
return [t, cf];
|
||||
}
|
||||
|
||||
public static double[] bdLatlng2Mercator(double lat, double lng) {
|
||||
double[] arr = null;
|
||||
double nLat = (lat > 74) ? 74 : lat;
|
||||
nLat = (nLat < -74) ? -74 : nLat;
|
||||
for (int i = 0; i < array1.length; i++) {
|
||||
if (lat >= array1[i]) {
|
||||
arr = array2[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (arr == null) {
|
||||
for (int i = array1.length - 1; i >= 0; i--) {
|
||||
if (lat <= -array1[i]) {
|
||||
arr = array2[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (arr == null) {
|
||||
throw new IllegalArgumentException("Invalid latitude");
|
||||
}
|
||||
double[] res = convertor(lng, lat, arr);
|
||||
return [res[0], res[1]]
|
||||
}
|
||||
|
||||
public static double[] bdMercator2Latlng(double x, double y) {
|
||||
double[] arr = null;
|
||||
double npX = Math.abs(x);
|
||||
double npY = Math.abs(y);
|
||||
for (int i = 0; i < array3.length; i++) {
|
||||
if (npY >= array3[i]) {
|
||||
arr = array4[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (arr == null) {
|
||||
throw new IllegalArgumentException("Invalid y value");
|
||||
}
|
||||
double[] res = convertor(npX, npY, arr);
|
||||
return [res[0], res[1]]
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
// Example usage
|
||||
double latitude = 4832281.76
|
||||
double longitude = 12959065.3
|
||||
|
||||
// double[] mercatorPoint = bdLatlng2Mercator(latitude, longitude);
|
||||
// System.out.println("Mercator Point: " + Arrays.toString(mercatorPoint));
|
||||
|
||||
double[] latlngPoint = bdMercator2Latlng(longitude, latitude);
|
||||
System.out.println("Latlng Point: " + Arrays.toString(latlngPoint));
|
||||
}
|
||||
}
|
||||
+380
@@ -0,0 +1,380 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
|
||||
def filePath = "D:\\数据\\南京\\南京银行测试-新跑.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
def communityObj = null
|
||||
if(row['E']){
|
||||
communityObj = findCommunityTown(community_CONN, row['C'], row['D'], row['E'])
|
||||
}
|
||||
if(communityObj){
|
||||
row['M'] = communityObj.province_code
|
||||
row['N'] = communityObj.city_code
|
||||
row['O'] = communityObj.area_code
|
||||
row['P'] = communityObj.street_code
|
||||
}
|
||||
if(row['D']){
|
||||
if(row['D'].toString().endsWith("区")){
|
||||
row['F'] = "非农"
|
||||
}else{
|
||||
if(communityObj){
|
||||
if(communityObj.has_gov && communityObj.has_gov=="是"){
|
||||
row['I'] = "是"
|
||||
row['F'] = "非农"
|
||||
}else{
|
||||
row['I'] = "否"
|
||||
if(row['E']){
|
||||
def townObj = isTown(row['C'], row['D'], row['E'])
|
||||
if(townObj){
|
||||
row['J'] = "是"
|
||||
row['F'] = "非农"
|
||||
}else{
|
||||
row['F'] = "农"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
//def filePath = "D:\\数据\\光大\\全量.xlsx"
|
||||
//ExcelUtils.modify(filePath, { row ->
|
||||
// if(!row['O']){
|
||||
// def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A'])
|
||||
// if(jsonObj && jsonObj.result){
|
||||
// row['B'] = jsonObj.result.countyLevelCity
|
||||
// if(row['B']){
|
||||
// if(row['B']=="是"){
|
||||
// row['C'] = "否"
|
||||
// }else if(row['B']=="否"){
|
||||
// row['C'] = "是"
|
||||
// }
|
||||
// }
|
||||
// row['D'] = jsonObj.result.district
|
||||
// row['E'] = jsonObj.result.districtCode
|
||||
// if(jsonObj.result.administrativeTowns && jsonObj.result.administrativeTowns=="是"){
|
||||
// row['F'] = jsonObj.result.administrativeTowns
|
||||
// row['G'] = jsonObj.result.town
|
||||
// row['H'] = jsonObj.result.townCode
|
||||
// }
|
||||
//
|
||||
// if(jsonObj.result.countyGov && jsonObj.result.countyGov=="是"){
|
||||
// row['I'] = jsonObj.result.countyGov
|
||||
// row['J'] = jsonObj.result.town
|
||||
// row['K'] = jsonObj.result.townCode
|
||||
// }
|
||||
//
|
||||
// if(jsonObj.result.devDistrict && jsonObj.result.devDistrict=="是"){
|
||||
// row['L'] = jsonObj.result.devDistrict
|
||||
// row['M'] = jsonObj.result.devDistrictName
|
||||
// }
|
||||
// row['N'] = jsonObj.result.addrVerify
|
||||
// row['O'] = jsonObj.result.province
|
||||
// row['P'] = jsonObj.result.city
|
||||
// row['Q'] = jsonObj.result.committee
|
||||
// row['R'] = jsonObj.result.typeName
|
||||
// row['S'] = jsonObj.result.splitCommittee
|
||||
// row['T'] = jsonObj.result.splitStreet
|
||||
// row['U'] = jsonObj.result.splitStreetNum
|
||||
// row['V'] = jsonObj.result.poiName
|
||||
// row['W'] = jsonObj.result.dev
|
||||
// }
|
||||
// }
|
||||
|
||||
// def retMap = SignatureUtils.getAliAddressStrc(row['A'])
|
||||
// println row['A'] +" " + retMap.toString()
|
||||
// if(retMap){
|
||||
// row['Y'] = retMap.prov
|
||||
// row['Z'] = retMap.city
|
||||
// row['AA'] = retMap.district
|
||||
// row['AB'] = retMap.town
|
||||
// row['AC'] = retMap.devzone
|
||||
// row['AD'] = retMap.road
|
||||
// row['AE'] = retMap.roadno
|
||||
// row['AF'] = retMap.poi
|
||||
// row['AG'] = retMap.subpoi
|
||||
// }
|
||||
// })
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
//
|
||||
// if(row['AA'] && !row['AO']){
|
||||
// if(row['Y']){
|
||||
// if(row['Y'].endsWith("区")){
|
||||
// row['AE'] = "否"
|
||||
// row['AF'] = "是"
|
||||
// }else {
|
||||
// row['AE'] = "是"
|
||||
// row['AF'] = "否"
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(row['AA']){
|
||||
// def dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], row['AA'])
|
||||
// if(!dev_name && row['AA'].contains("经济开发区")){
|
||||
// dev_name = row['AA'].replace("经济开发区","经济技术开发区")
|
||||
// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
|
||||
// }
|
||||
// if(!dev_name && row['AA'].contains("经济技术开发区")){
|
||||
// dev_name = row['AA'].replace("经济技术开发区","经济开发区")
|
||||
// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
|
||||
// }
|
||||
// if(dev_name){
|
||||
// row['AO'] = "是"
|
||||
// row['AP'] = dev_name
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(row['W'] && row['X'] && row['Y'] && row['Z']){
|
||||
// def townObj = isTown(row['X'], row['Y'], row['Z'])
|
||||
// if(townObj){
|
||||
// row['AI'] = "是"
|
||||
// row['AJ'] = row['Z']
|
||||
// }
|
||||
//
|
||||
// def communityObj = findCommunityTown(community_CONN, row['X'], row['Y'], row['Z'])
|
||||
// if(communityObj){
|
||||
// row['AL'] = isGov(communityObj)
|
||||
// if(row['AL'] && row['AL'] =="是"){
|
||||
// row['AM'] = row['Z']
|
||||
// row['AN'] = communityObj.street_code
|
||||
// }
|
||||
// row['AG'] = communityObj.area_name
|
||||
// row['AH'] = communityObj.area_code
|
||||
//
|
||||
// if(row['AI'] && row['AI'] == "是"){
|
||||
// row['AK'] = communityObj.street_code
|
||||
// }
|
||||
// }else{
|
||||
// def towns = findAllTowns(community_CONN, row['X'], row['Y'])
|
||||
// def retObj = findBestSimilar(row['Z'], towns)
|
||||
// if(retObj && retObj.score>=0.5){
|
||||
// row['AQ'] = retObj.name
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
// if(row['F'] && !row['T']){
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("province", row['B'])
|
||||
// def rets = development_zone_CONN.find(q).toList()
|
||||
// def maxScore = 0;
|
||||
// def map = [:]
|
||||
// if(rets){
|
||||
// for(def r : rets){
|
||||
// def score1 = CosineSimilarAlgorithm.getSimilarityRatio(row['B']+row['C']+row['F'], r.name);
|
||||
// if(score1>maxScore){
|
||||
// map.name = r.name
|
||||
// map.score = score1
|
||||
// maxScore = score1
|
||||
// }
|
||||
// }
|
||||
// if(map){
|
||||
// row['W'] = map.name
|
||||
// row['X'] = map.score
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(row['C'] && row['R']){
|
||||
def obj = findFirstData(community_CONN, row['C'], row['D'], row['R'])
|
||||
if(obj){
|
||||
row['S'] = obj.street_code
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
def findBestSimilar(town, towns){
|
||||
def map = [:];
|
||||
def maxScore = 0
|
||||
for(def name : towns){
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(town, name);
|
||||
if(score1>maxScore){
|
||||
map.name = name
|
||||
map.score = score1
|
||||
maxScore = score1
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def isDev(development_zone_CONN, def prov, def city, def region, def dev){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", prov)
|
||||
dev = dev.replace("(","").replace(")","").replace("市","")
|
||||
if(dev.contains(region) && region.toString().endsWith("区")){
|
||||
def newRegion = region.replace("区","")
|
||||
dev = dev.replace(region, newRegion)
|
||||
}
|
||||
Pattern pattern = Pattern.compile('^.*' + dev + '.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("name",new BasicDBObject('$regex', pattern))
|
||||
def rets = development_zone_CONN.find(q).toList()
|
||||
def developmentZone = null
|
||||
if(rets){
|
||||
def devList = []
|
||||
devList.add(dev);
|
||||
try{
|
||||
devList.add(region.replace("县","").replace("区","").replace("市","") + dev);
|
||||
}catch(e){
|
||||
println region +" "+ e.getMessage()
|
||||
}
|
||||
try{
|
||||
devList.add(city.replace("市","") + region.replace("县","").replace("区","").replace("市","") + dev);
|
||||
}catch(e){
|
||||
println city+" "+region+" "+e.getMessage()
|
||||
}
|
||||
try{
|
||||
devList.add(prov.replace("省","") + city.replace("市","")+ region.replace("县","").replace("区","").replace("市","") + dev);
|
||||
}catch(e){
|
||||
println prov+" "+city+" "+region+" "+e.getMessage()
|
||||
}
|
||||
try{
|
||||
devList.add(city.replace("市","") + dev);
|
||||
}catch(e){
|
||||
println e
|
||||
}
|
||||
try{
|
||||
devList.add(prov.replace("省","")+ city.replace("市","") + dev);
|
||||
}catch(e){
|
||||
println e
|
||||
}
|
||||
try{
|
||||
devList.add(prov.replace("省","")+ region.replace("县","").replace("区","").replace("市","")+ dev);
|
||||
}catch(e){
|
||||
println e
|
||||
}
|
||||
devList.add(prov.replace("省","")+ dev);
|
||||
|
||||
for(def zone : rets){
|
||||
String name = zone.name.replace("省","").replace("市","").replace("县","")
|
||||
if(!name.endsWith("区")){
|
||||
name = name.replace("区","");
|
||||
}
|
||||
if(devList.contains(name)){
|
||||
developmentZone = zone;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(developmentZone){
|
||||
return developmentZone.name;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def isTown( city, region, town){
|
||||
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
|
||||
// obj = it
|
||||
// },"192.168.0.89")
|
||||
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool":{
|
||||
"must":[
|
||||
{"term": {"city.keyword":"${city}"}},
|
||||
{"term": {"district.keyword":"${region}"}},
|
||||
{"term": {"town.keyword":"${town}"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
def obj = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
obj = hits[0]
|
||||
}
|
||||
}
|
||||
println(response)
|
||||
return obj
|
||||
}
|
||||
|
||||
def findCommunityTown(community_CONN, city, region, town){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", city)
|
||||
q.put("area_name", region)
|
||||
Pattern pattern = Pattern.compile(town + '.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("street_name",new BasicDBObject('$regex', pattern))
|
||||
def rets = community_CONN.find(q).toList()
|
||||
def obj = null
|
||||
if(rets){
|
||||
obj = rets[0]
|
||||
}
|
||||
|
||||
return obj;
|
||||
|
||||
}
|
||||
|
||||
def findAllTowns(community_CONN, city, region){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", city)
|
||||
q.put("area_name", region)
|
||||
def rets = community_CONN.find(q).toList()
|
||||
def towns = []
|
||||
for(def t : rets){
|
||||
if(!towns.contains(t.street_name)){
|
||||
towns << t.street_name
|
||||
}
|
||||
}
|
||||
return towns;
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findFirstData(community_CONN, city, region, town){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", city)
|
||||
q.put("area_name", region)
|
||||
q.put("street_name", town)
|
||||
def obj = community_CONN.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def isGov(obj){
|
||||
String hasGovFlag = null;
|
||||
if(obj){
|
||||
hasGovFlag = obj.has_gov;
|
||||
}
|
||||
if(StringUtils.isNotBlank(hasGovFlag) && hasGovFlag.equals("是")){
|
||||
return "是"
|
||||
}else {
|
||||
return "否"
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.fangjia.script.gj.village.ailiyun
|
||||
|
||||
import com.aliyun.tea.*;
|
||||
|
||||
class AnliyunApiUtils {
|
||||
|
||||
/**
|
||||
* 使用AK&SK初始化账号Client
|
||||
* @param accessKeyId
|
||||
* @param accessKeySecret
|
||||
* @return Client
|
||||
* @throws Exception
|
||||
*/
|
||||
public static com.aliyun.iot20180120.Client createClient(String accessKeyId, String accessKeySecret) throws Exception {
|
||||
com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config()
|
||||
// 必填,您的 AccessKey ID
|
||||
.setAccessKeyId(accessKeyId)
|
||||
// 必填,您的 AccessKey Secret
|
||||
.setAccessKeySecret(accessKeySecret);
|
||||
// Endpoint 请参考 https://api.aliyun.com/product/Iot
|
||||
config.endpoint = "iot.cn-beijing.aliyuncs.com";
|
||||
return new com.aliyun.iot20180120.Client(config);
|
||||
}
|
||||
|
||||
public static void main(String[] args_) throws Exception {
|
||||
java.util.List<String> args = java.util.Arrays.asList(args_);
|
||||
// 请确保代码运行环境设置了环境变量 ALIBABA_CLOUD_ACCESS_KEY_ID 和 ALIBABA_CLOUD_ACCESS_KEY_SECRET。
|
||||
// 工程代码泄露可能会导致 AccessKey 泄露,并威胁账号下所有资源的安全性。以下代码示例使用环境变量获取 AccessKey 的方式进行调用,仅供参考,建议使用更安全的 STS 方式,更多鉴权访问方式请参见:https://help.aliyun.com/document_detail/378657.html
|
||||
com.aliyun.iot20180120.Client client = createClient(Config.accessKey, Config.accessKeySecret);
|
||||
com.aliyun.iot20180120.models.CreateDeviceGroupRequest createDeviceGroupRequest = new com.aliyun.iot20180120.models.CreateDeviceGroupRequest()
|
||||
.setIotInstanceId("your_value")
|
||||
.setSuperGroupId("your_value")
|
||||
.setGroupDesc("your_value")
|
||||
.setGroupName("your_value");
|
||||
try {
|
||||
// 复制代码运行请自行打印 API 的返回值
|
||||
client.createDeviceGroupWithOptions(createDeviceGroupRequest, new com.aliyun.teautil.models.RuntimeOptions());
|
||||
} catch (TeaException error) {
|
||||
// 错误 message
|
||||
System.out.println(error.getMessage());
|
||||
// 诊断地址
|
||||
System.out.println(error.getData().get("Recommend"));
|
||||
com.aliyun.teautil.Common.assertAsString(error.message);
|
||||
} catch (Exception _error) {
|
||||
TeaException error = new TeaException(_error.getMessage(), _error);
|
||||
// 错误 message
|
||||
System.out.println(error.getMessage());
|
||||
// 诊断地址
|
||||
System.out.println(error.getData().get("Recommend"));
|
||||
com.aliyun.teautil.Common.assertAsString(error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.fangjia.script.gj.village.ailiyun
|
||||
|
||||
/* 服务端API签名配置文件
|
||||
*
|
||||
* @author: ali
|
||||
* @version: 0.1 2018-08-08 08:23:54
|
||||
*/
|
||||
public class Config {
|
||||
// AccessKey信息
|
||||
public static String accessKey = "LTAI5t7UKjXaoGxzet2Vh3bB";
|
||||
public static String accessKeySecret = "2avjPxmI7FlIRHvMC5C1UuAnd9iG8z";
|
||||
|
||||
public final static String CHARSET_UTF8 = "utf8";
|
||||
}
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
package com.fangjia.script.gj.village.ailiyun
|
||||
|
||||
import com.aliyuncs.CommonRequest
|
||||
import com.aliyuncs.CommonResponse
|
||||
import com.aliyuncs.DefaultAcsClient
|
||||
import com.aliyuncs.IAcsClient
|
||||
import com.aliyuncs.exceptions.ClientException
|
||||
import com.aliyuncs.http.FormatType
|
||||
import com.aliyuncs.http.MethodType
|
||||
import com.aliyuncs.profile.DefaultProfile
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URLDecoder;
|
||||
import java.net.URLEncoder
|
||||
import java.rmi.ServerException;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* 服务端API签名
|
||||
*
|
||||
* @author: ali
|
||||
* @version: 0.1 2018-06-21 20:47:05
|
||||
*/
|
||||
public class SignatureUtils {
|
||||
|
||||
private final static String CHARSET_UTF8 = "utf8";
|
||||
private final static String ALGORITHM = "HmacSHA1";
|
||||
private final static String SEPARATOR = "&";
|
||||
|
||||
public static Map<String, String> splitQueryString(String url)
|
||||
throws URISyntaxException, UnsupportedEncodingException {
|
||||
URI uri = new URI(url);
|
||||
String query = uri.getQuery();
|
||||
final String[] pairs = query.split("&");
|
||||
TreeMap<String, String> queryMap = new TreeMap<String, String>();
|
||||
for (String pair : pairs) {
|
||||
final int idx = pair.indexOf("=");
|
||||
final String key = idx > 0 ? pair.substring(0, idx) : pair;
|
||||
if (!queryMap.containsKey(key)) {
|
||||
queryMap.put(key, URLDecoder.decode(pair.substring(idx + 1), CHARSET_UTF8));
|
||||
}
|
||||
}
|
||||
return queryMap;
|
||||
}
|
||||
|
||||
public static String generate(String method, Map<String, String> parameter, String accessKeySecret)
|
||||
throws Exception {
|
||||
String signString = generateSignString(method, parameter);
|
||||
System.out.println("signString---" + signString);
|
||||
byte[] signBytes = hmacSHA1Signature(accessKeySecret + "&", signString);
|
||||
String signature = newStringByBase64(signBytes);
|
||||
System.out.println("signature----" + signature);
|
||||
if ("POST".equals(method))
|
||||
return signature;
|
||||
return URLEncoder.encode(signature, "UTF-8");
|
||||
}
|
||||
|
||||
public static String generateSignString(String httpMethod, Map<String, String> parameter) throws IOException {
|
||||
TreeMap<String, String> sortParameter = new TreeMap<String, String>();
|
||||
sortParameter.putAll(parameter);
|
||||
String canonicalizedQueryString = UrlUtil.generateQueryString(sortParameter, true);
|
||||
if (null == httpMethod) {
|
||||
throw new RuntimeException("httpMethod can not be empty");
|
||||
}
|
||||
StringBuilder stringToSign = new StringBuilder();
|
||||
stringToSign.append(httpMethod).append(SEPARATOR);
|
||||
stringToSign.append(percentEncode("/")).append(SEPARATOR);
|
||||
stringToSign.append(percentEncode(canonicalizedQueryString));
|
||||
return stringToSign.toString();
|
||||
}
|
||||
|
||||
public static String percentEncode(String value) {
|
||||
try {
|
||||
return value == null ? null
|
||||
: URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
|
||||
"~");
|
||||
} catch (Exception e) {
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
public static byte[] hmacSHA1Signature(String secret, String baseString) throws Exception {
|
||||
if (StringUtils.isEmpty(secret)) {
|
||||
throw new IOException("secret can not be empty");
|
||||
}
|
||||
if (StringUtils.isEmpty(baseString)) {
|
||||
return null;
|
||||
}
|
||||
Mac mac = Mac.getInstance("HmacSHA1");
|
||||
SecretKeySpec keySpec = new SecretKeySpec(secret.getBytes(CHARSET_UTF8), ALGORITHM);
|
||||
mac.init(keySpec);
|
||||
return mac.doFinal(baseString.getBytes(CHARSET_UTF8));
|
||||
}
|
||||
|
||||
public static String newStringByBase64(byte[] bytes) throws UnsupportedEncodingException {
|
||||
if (bytes == null || bytes.length == 0) {
|
||||
return null;
|
||||
}
|
||||
return new String(Base64.encodeBase64(bytes, false), CHARSET_UTF8);
|
||||
}
|
||||
|
||||
|
||||
// public static void main(String[] a){
|
||||
// Map<String, String> map = new HashMap<String, String>();
|
||||
// Calendar calendar = Calendar.getInstance()
|
||||
// def startTime = DateUtil.formatByPattern(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")
|
||||
// // 公共参数
|
||||
// map.put("Format", "JSON");
|
||||
// map.put("Version", "2019-11-18");
|
||||
// map.put("AccessKeyId", Config.accessKey);
|
||||
// map.put("SignatureMethod", "HMAC-SHA1");
|
||||
// map.put("Timestamp", startTime);
|
||||
// map.put("SignatureVersion", "1.0");
|
||||
// map.put("SignatureNonce", System.currentTimeMillis().toString());
|
||||
//// map.put("RegionId", "cn-shanghai");
|
||||
// // 请求参数
|
||||
// map.put("Action", "StreetStd");
|
||||
// map.put("ServiceCode", "addrp");
|
||||
// map.put("AppKey", "Xczmmbxlur2g");
|
||||
// map.put("Text","山东省烟台市莱阳市龙门东路1号0027、0028")
|
||||
// try {
|
||||
// String signature = SignatureUtils.generate("GET", map, Config.accessKeySecret);
|
||||
// if(signature){
|
||||
// map.put("signature", signature);
|
||||
// }
|
||||
// String content = new HttpRequest().get("https://address-purification.cn-hangzhou.aliyuncs.com", map, "utf-8")
|
||||
// JSONObject result = JSONObject.fromObject(content)
|
||||
// System.out.println("最终signature: " + signature);
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// System.out.println();
|
||||
// }
|
||||
|
||||
|
||||
public static def getAliAddressStrc(def address) {
|
||||
DefaultProfile defaultProfile = DefaultProfile.getProfile(
|
||||
"cn-hangzhou",
|
||||
Config.accessKey,
|
||||
Config.accessKeySecret);
|
||||
IAcsClient client = new DefaultAcsClient(defaultProfile);
|
||||
// 创建API请求并设置参数
|
||||
CommonRequest request = new CommonRequest();
|
||||
request.setSysDomain("address-purification.cn-hangzhou.aliyuncs.com");
|
||||
request.setSysVersion("2019-11-18");
|
||||
request.setSysProduct("address-purification");
|
||||
request.setSysLocationProduct("addrp");
|
||||
//设置要请求的服务
|
||||
request.setSysAction("StreetStd");
|
||||
//设置请求方式
|
||||
request.setSysMethod(MethodType.POST);
|
||||
request.setHttpContentType(FormatType.FORM);
|
||||
//请求头
|
||||
request.putHeadParameter("x-acs-signature-method", "HMAC-SHA1");
|
||||
request.putHeadParameter("x-acs-signature-nonce", UUID.randomUUID().toString());
|
||||
//设置项目token
|
||||
request.putBodyParameter("AppKey", "Xczmmbxlur2g");
|
||||
//设置请求参数
|
||||
request.putBodyParameter("Text", address);
|
||||
// request.putBodyParameter("DefaultCity", "杭州市");
|
||||
request.putQueryParameter("AcceptLanguage", "zh-CN");
|
||||
|
||||
try {
|
||||
CommonResponse response = client.getCommonResponse(request);
|
||||
def data = response.getData();
|
||||
def result = JSONObject.fromObject(data)
|
||||
if(result && result.Data){
|
||||
def data1 = JSONObject.fromObject(result.Data)
|
||||
def std = data1.street_std
|
||||
if(std){
|
||||
def arrs = std.split("\t")
|
||||
def map = [:]
|
||||
for(def s : arrs){
|
||||
def paramValues = s.split("=")
|
||||
map.put(paramValues[0], paramValues[1])
|
||||
}
|
||||
return map
|
||||
}
|
||||
}
|
||||
|
||||
} catch (ServerException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (ClientException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
package com.fangjia.script.gj.village.ailiyun
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* URL处理类
|
||||
*
|
||||
* @author: ali
|
||||
* @version: 0.1 2018-06-21 20:40:52
|
||||
*/
|
||||
public class UrlUtil {
|
||||
|
||||
private final static String CHARSET_UTF8 = "utf8";
|
||||
|
||||
public static String urlEncode(String url) {
|
||||
if (!StringUtils.isEmpty(url)) {
|
||||
try {
|
||||
url = URLEncoder.encode(url, "UTF-8");
|
||||
} catch (Exception e) {
|
||||
System.out.println("Url encode error:" + e.getMessage());
|
||||
}
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
public static String generateQueryString(Map<String, String> params, boolean isEncodeKV) {
|
||||
StringBuilder canonicalizedQueryString = new StringBuilder();
|
||||
for (Map.Entry<String, String> entry : params.entrySet()) {
|
||||
if (isEncodeKV)
|
||||
canonicalizedQueryString.append(percentEncode(entry.getKey())).append("=")
|
||||
.append(percentEncode(entry.getValue())).append("&");
|
||||
else
|
||||
canonicalizedQueryString.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
|
||||
}
|
||||
if (canonicalizedQueryString.length() > 1) {
|
||||
canonicalizedQueryString.setLength(canonicalizedQueryString.length() - 1);
|
||||
}
|
||||
return canonicalizedQueryString.toString();
|
||||
}
|
||||
|
||||
public static String percentEncode(String value) {
|
||||
try {
|
||||
// 使用URLEncoder.encode编码后,将"+","*","%7E"做替换即满足API规定的编码规范。
|
||||
return value == null ? null
|
||||
: URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
|
||||
"~");
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return "";
|
||||
}
|
||||
}
|
||||
+172
File diff suppressed because one or more lines are too long
+149
@@ -0,0 +1,149 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
|
||||
//def s = getTownshipInfo("118.702988,39.740979")
|
||||
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
//// if(!row['I']){
|
||||
//// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
||||
//// if(rets){
|
||||
//// def obj = findOneByName(rets, row['C']+"人民政府")
|
||||
//// if(obj){
|
||||
//// row['I'] = obj.lng
|
||||
//// row['J'] = obj.lat
|
||||
//// println row['C'] + " "+obj
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// if(row['D'] && !row['F']){
|
||||
//// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||
//// def map = ["lon":row['D'], "lat":row['E']]
|
||||
//// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
|
||||
//// row['F'] = township
|
||||
//// }
|
||||
//
|
||||
//// if(row['F'] && !row['J']){
|
||||
//// def q = new BasicDBObject()
|
||||
//// q.put("city_name", row['B'])
|
||||
//// q.put("area_name", row['C'])
|
||||
//// def streetName = row['F']
|
||||
//// q.put("street_name", streetName)
|
||||
//// def count = COMMUNITY_CONN.count(q);
|
||||
//// row['J'] = count
|
||||
////// row['H'] = streetName
|
||||
//// }
|
||||
//// if(!row['H']){
|
||||
//// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
|
||||
//// if(gaodeMap && gaodeMap.township){
|
||||
//// row['H'] = gaodeMap.township
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// if(row['J'] && row['J'].toString().toDouble()>0){
|
||||
//// def up_query = new BasicDBObject()
|
||||
//// up_query.put("city_name", row['B'])
|
||||
//// up_query.put("area_name", row['C'])
|
||||
//// up_query.put("street_name", row['F'])
|
||||
//// def udata = new BasicDBObject()
|
||||
//// udata.put("has_gov", "是")
|
||||
//// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
|
||||
//// }
|
||||
//
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
for(def d : objs){
|
||||
if(!d.name_pinyin){
|
||||
if(d.name){
|
||||
println d._id
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_pinyin", py1)
|
||||
if(d.location){
|
||||
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
|
||||
udata.put("location", ["lat":lat, "lon":lon])
|
||||
}
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.district = district
|
||||
map.township = township
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findOneByName(def rets, name){
|
||||
def obj = null
|
||||
for(def r : rets){
|
||||
if(r.name == name){
|
||||
obj = r
|
||||
break;
|
||||
}
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def findTownRange(city, TOWN_POLYLINECOLL){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community_new_2023", MONGO100_URL)
|
||||
|
||||
filePath = "D:\\Êý¾Ý\\ɿũ\\18Íò×îÖÕ-id.xlsx"
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify(filePath, { d->
|
||||
|
||||
if(d['A']){
|
||||
def q = new BasicDBObject("_id", d['A'])
|
||||
def dbo = COMMUNITY_CONN.findOne(q)
|
||||
if(dbo){
|
||||
index++
|
||||
println index
|
||||
COMMUNITY_CONN.remove(q)
|
||||
}else{
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("_id" , d['A'])
|
||||
// udata.put("province" , d['A'])
|
||||
// udata.put("province_code" , d['A'])
|
||||
// udata.put("city_code" , d['A'])
|
||||
// udata.put("area_name" , d['A'])
|
||||
// udata.put("area_code" , d['A'])
|
||||
// udata.put("street_name" , d['A'])
|
||||
// udata.put("street_code" , d['A'])
|
||||
// udata.put("name" , d['A'])
|
||||
// udata.put("code" , d['A'])
|
||||
// udata.put("type_code" , d['A'])
|
||||
// udata.put("type_name" , d['A'])
|
||||
// udata.put("location_type" , d['A'])
|
||||
// udata.put("geohash" , d['A'])
|
||||
// udata.put("location" , d['A'])
|
||||
// udata.put("new_poi_name" , d['A'])
|
||||
// udata.put("new_area_name" , d['A'])
|
||||
// udata.put("city_name" , d['A'])
|
||||
// COMMUNITY_CONN.save(udata);
|
||||
}
|
||||
}
|
||||
})
|
||||
+562
@@ -0,0 +1,562 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 厦门银行
|
||||
* 福建省,重庆市poi数据处理
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static def version = new Date().getTime()
|
||||
|
||||
public static def TAGS = ["加油站","其它能源站","加气站","洗车场","充电站","餐饮相关场所","休闲餐饮场所","购物相关场所","商场","超级市场","家电电子卖场","花鸟鱼虫市场","家居建材市场","综合市场","特色商业街","生活服务场所","邮局","事务所","人才市场","洗浴推拿场所","体育休闲服务场所","运动场馆","娱乐场所","度假疗养场所","休闲场所","医疗保健服务场所","综合医院","专科医院","诊所","急救中心","疾病预防机构","住宿服务相关","宾馆酒店","旅馆招待所","风景名胜相关","公园广场","风景名胜","产业园区","政府及社会团体相关","政府机关","外国机构","社会团体","公检法机构","交通车辆管理","工商税务机构","科教文化场所","博物馆","展览馆","会展中心","美术馆","图书馆","科技馆","天文馆","文化宫","档案馆","文艺团体","传媒机构","学校","科研机构","培训机构","驾校","机场相关","银行","银行相关","保险公司","证券公司","财务公司","公司企业","知名企业","公司","工厂","农林牧渔基地","收费站","服务区","标志性建筑物","地名地址信息","热点地名","紧急避难场所"]
|
||||
|
||||
|
||||
public final static DBCollection COMPANY_206_COLL = MongoUtils.getCollectionByName("business", "company_huzhou", "192.168.0.206",27020)
|
||||
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
|
||||
|
||||
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
|
||||
|
||||
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
def fullCityMap = null
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GaoDePoiDataProcess(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// println ProvinceData.getProvince("成都")
|
||||
// synchMapPoi();
|
||||
|
||||
|
||||
synchCompany()
|
||||
// poiCatProcess()
|
||||
// addressAndCatProcess()
|
||||
// synchNetwork()
|
||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||
// sysnchPoi()
|
||||
}
|
||||
|
||||
def sysnchPoi(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("version", new BasicDBObject('$gte',1701273600000))
|
||||
def dbcursor = MAPPOI_CONN_TEST.find(q).batchSize(20000)
|
||||
def ret = []
|
||||
def i = 0
|
||||
for(def d : dbcursor){
|
||||
i++
|
||||
ret << d
|
||||
if(ret.size()%50000==0){
|
||||
println "第 "+(i/50000)+" 个"+ret.size()
|
||||
MAPPOI_CONN.insert(ret);
|
||||
ret = [];
|
||||
}
|
||||
}
|
||||
|
||||
if(ret.size()>0){
|
||||
MAPPOI_CONN.insert(ret);
|
||||
}
|
||||
}
|
||||
|
||||
def synchCompany(){
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
|
||||
def citys = ["杭州","宁波","温州","嘉兴","台州","绍兴","金华","舟山","衢州","丽水"]
|
||||
for(def sampleCity : citys){
|
||||
|
||||
def cityObj = fullCityMap.get(sampleCity);
|
||||
def city = null
|
||||
if(cityObj){
|
||||
city = cityObj.city
|
||||
}
|
||||
def townMap = findTownRange(city)
|
||||
def companyDatas = findCompanyDatas(city)
|
||||
println city+ " 加载公司数据:"+companyDatas.size()
|
||||
def rows = running(city, companyDatas, "company", townMap)
|
||||
save(city, rows)
|
||||
// saveToMG(city, rows)
|
||||
println "完成"
|
||||
}
|
||||
}
|
||||
|
||||
def findCompanyDatas(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("done", 1)
|
||||
q.put("geo_city", city)
|
||||
// q.put("geo_level", new BasicDBObject('$in', ['门牌号','兴趣点']))
|
||||
return COMPANY_206_COLL.find(q).toList()
|
||||
}
|
||||
|
||||
def synchMapPoi(){
|
||||
def runedCity = []
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
def citys = CommonDatautils.findAllCity().reverse();
|
||||
for(def sampleCity : citys){
|
||||
def cityObj = fullCityMap.get(sampleCity);
|
||||
def city = null
|
||||
if(cityObj){
|
||||
city = cityObj.city
|
||||
}
|
||||
if(city && !runedCity.contains(city)){
|
||||
def townMap = findTownRange(city)
|
||||
//// def results = CommonDatautils.findAmapDetailBySql(city)
|
||||
def results = CommonDatautils.findNewAmapDetailBySql(city)
|
||||
println "city: "+city +" town: "+townMap.size()+" poi:"+results.size()
|
||||
def rows = running(city, results, "map", townMap)
|
||||
//// save(city, rows)
|
||||
saveToMG(city, rows)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def running(city,results, type, townMap){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def prov = ProvinceData.getProvince(city.replace("市",""))
|
||||
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(type == "map"){
|
||||
rets.addAll(createData(prov, map, townMap))
|
||||
}else if(type == "company"){
|
||||
rets.addAll(createCompanyData(map, townMap))
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createCompanyData(map, townMap){
|
||||
def retMap = [:]
|
||||
retMap.data_id = map._id.toString()
|
||||
retMap.fjw_province = map.province_name
|
||||
retMap.fjw_city = map.city_name
|
||||
|
||||
try{
|
||||
def fjw_region = RegionData.formatRegion(map.city_name, map.geo_district)
|
||||
if(fjw_region){
|
||||
retMap.fjw_district = fjw_region
|
||||
}
|
||||
}catch(Exception e){
|
||||
println map.geo_district
|
||||
}
|
||||
|
||||
|
||||
if(!retMap.fjw_district ){
|
||||
retMap.fjw_district = RegionData.formatRegion(map.city_name, map.region_name)
|
||||
}
|
||||
retMap.province = map.geo_province
|
||||
retMap.city = map.geo_city
|
||||
retMap.district = map.geo_district
|
||||
retMap.cat = "企业"
|
||||
retMap.name = map.ent_name
|
||||
// def pointList = townMap.get(retMap.fjw_district)
|
||||
|
||||
def locationStr = map.geo_location
|
||||
if(locationStr){
|
||||
def arrs = locationStr.split(",")
|
||||
|
||||
retMap.township = InsidePolygon.isInArea2(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), townMap)
|
||||
|
||||
def gd_location = [:]
|
||||
gd_location.lon = DataHandleUtils.getDoubleRound(6, arrs[0].toString().toDouble())
|
||||
gd_location.lat = DataHandleUtils.getDoubleRound(6, arrs[1].toString().toDouble())
|
||||
retMap.location = gd_location
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
|
||||
retMap.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
|
||||
retMap.gps_location = gpslocation
|
||||
}
|
||||
|
||||
if(!retMap.township){
|
||||
def gaodeMap = getTownshipInfo(locationStr)
|
||||
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
retMap.township = gaodeMap.township
|
||||
}
|
||||
|
||||
if(!retMap.fjw_district){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district)
|
||||
}
|
||||
|
||||
}
|
||||
if(!fjw_region && !retMap.fjw_district){
|
||||
println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap
|
||||
}
|
||||
retMap.fjw_district = fjw_region
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
retMap.address = map.reg_addr
|
||||
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.ent_name, " ")
|
||||
retMap.version = version
|
||||
retMap.status = 1
|
||||
retMap.credit_code = map.unified_code
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def createData(province, map, townMap){
|
||||
def fjw_province = province
|
||||
def fjw_city = RegionData.formatCity(map.city)
|
||||
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
|
||||
map.fjw_province = fjw_province
|
||||
map.fjw_city = fjw_city
|
||||
|
||||
if(map.cat){
|
||||
def arrs = map.cat.split(";|:")
|
||||
map.type = arrs[0]
|
||||
if(arrs.size()>=2){
|
||||
map.secondary_type = arrs[1]
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
map.small_type = arrs[2]
|
||||
}
|
||||
if(arrs.size()>3){
|
||||
map.cat = arrs[0]+";"+arrs[1]+";"+arrs[2]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
|
||||
if(!map.township){
|
||||
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
|
||||
if(!fjw_region){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
}
|
||||
}
|
||||
|
||||
if(!fjw_region){
|
||||
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
|
||||
}
|
||||
|
||||
map.fjw_district = fjw_region
|
||||
}
|
||||
map.fjw_district = fjw_region
|
||||
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
|
||||
map.version = version
|
||||
|
||||
if(map.secondary_type && TAGS.contains(map.secondary_type)){
|
||||
map.status = 0
|
||||
}else{
|
||||
map.status = 1
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def save(city,rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "[${city}] 结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def saveToMG(city, rows){
|
||||
|
||||
def rets = []
|
||||
for(def i=0;i<rows.size();i++){
|
||||
def m = rows[i]
|
||||
def obj = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
rets << obj
|
||||
if(i%50000==0){
|
||||
MAPPOI_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
MAPPOI_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.district = district
|
||||
map.township = township
|
||||
return map;
|
||||
}
|
||||
|
||||
def findTownRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
// def regionMap = townMap.get(d.fjw_region)
|
||||
// if(!regionMap){
|
||||
// regionMap = [:]
|
||||
// townMap.put(d.fjw_region, regionMap)
|
||||
// }
|
||||
// def townInfoMap = regionMap.get(d.town)
|
||||
// if(townInfoMap){
|
||||
// townInfoMap = [:]
|
||||
// regionMap.put()
|
||||
// }
|
||||
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
def poiCatProcess(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\数据.xlsx",{d->
|
||||
def cat = d['B']
|
||||
if(cat && cat.size()>50){
|
||||
println cat
|
||||
def arrs = cat.split(";")
|
||||
def newCat = ""
|
||||
if(arrs.size()>3){
|
||||
newCat = arrs[0]+";"+arrs[1]+";"+arrs[2]
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("cat",newCat)
|
||||
udata.put("type",arrs[0])
|
||||
udata.put("secondary_type",arrs[1])
|
||||
udata.put("small_type",arrs[2])
|
||||
MAPPOI_CONN.update(new BasicDBObject("cat", cat), new BasicDBObject('$set', udata), false, true)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addressAndCatProcess(){
|
||||
def datas = FileUtils.readTxt("C:\\Users\\gaojuan\\Downloads\\BIGDATA_POI_v1.bad")
|
||||
datas.each{d->
|
||||
def arrs = d.split("@\\|@")
|
||||
if(arrs){
|
||||
def udata = new BasicDBObject()
|
||||
// def address = arrs[5]
|
||||
// if(address.size()>=120){
|
||||
// udata.put("address", "")
|
||||
// }
|
||||
def cat = arrs[8]
|
||||
def catArrs = cat.split(";")
|
||||
if(catArrs.size()>3){
|
||||
cat = catArrs[0]+";"+catArrs[1]+";"+catArrs[2]
|
||||
if(cat.size()>=45){
|
||||
println "取前3依然超:"+cat
|
||||
}else{
|
||||
udata.put("type",catArrs[0])
|
||||
udata.put("secondary_type",catArrs[1])
|
||||
udata.put("small_type",catArrs[2])
|
||||
udata.put("cat", cat)
|
||||
}
|
||||
|
||||
}else{
|
||||
println cat
|
||||
}
|
||||
|
||||
if(udata){
|
||||
MAPPOI_CONN.update(new BasicDBObject("_id", arrs[0]), new BasicDBObject('$set', udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def synchNetwork(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx")
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def location = d['I']
|
||||
def locMap = [:]
|
||||
if(location){
|
||||
def arrs = location.split(",")
|
||||
locMap.lon = arrs[0].toString().toDouble()
|
||||
locMap.lat = arrs[1].toString().toDouble()
|
||||
}
|
||||
rets << ["id":new ObjectId().toString(), "name":d['B'], "address":d['C'], "location":locMap]
|
||||
}
|
||||
|
||||
saveEs("xm_network", rets)
|
||||
}
|
||||
|
||||
def saveEs(index,rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient("192.168.0.12")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest(index, "_doc", map.id).source(map, XContentType.JSON));
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class NetworkDistanceBatch extends BaseBatch{
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
|
||||
|
||||
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new NetworkDistanceBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def networks = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx", 0, 1)
|
||||
def citys = ["厦门","三明","宁德","泉州","福州","漳州","南平","龙岩","莆田","重庆"]
|
||||
for(def c : citys){
|
||||
run(c, "district", networks)
|
||||
}
|
||||
// saveMapData()
|
||||
}
|
||||
|
||||
def saveMapData(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\厦门银行鹭通支行.xlsx",{d->
|
||||
if(d['C']){
|
||||
def arrs = d['C'].split(",")
|
||||
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("network_name", d['B'])
|
||||
obj.put("ntk_gd_lng", arrs[0].toString().toDouble())
|
||||
obj.put("ntk_gd_lat", arrs[1].toString().toDouble())
|
||||
obj.put("ntk_bd_lng", lng2)
|
||||
obj.put("ntk_bd_lat", lat2)
|
||||
obj.put("comm_id", d['D'])
|
||||
obj.put("city", d['E'])
|
||||
obj.put("region", d['F'])
|
||||
obj.put("name", d['G'])
|
||||
obj.put("distance", d['I'].toString().toDouble().toInteger())
|
||||
obj.put("time", d['K'].toString().toDouble().toInteger())
|
||||
obj.put("gh_distance", d['J'].toString().toDouble().toInteger())
|
||||
def dist_loc_arr = d['H'].split(",")
|
||||
|
||||
def baidu_dist = BaiDuApiUtils.geoconv(dist_loc_arr[0].toString().toDouble(), dist_loc_arr[1].toString().toDouble(), 3)
|
||||
obj.put("dist_gd_lng", dist_loc_arr[0].toString().toDouble())
|
||||
obj.put("dist_gd_lat", dist_loc_arr[1].toString().toDouble())
|
||||
obj.put("dist_bd_lng", baidu_dist.x)
|
||||
obj.put("dist_bd_lat", baidu_dist.y)
|
||||
|
||||
MAP_NETWORK_COLL.save(obj)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(city, cat, networks){
|
||||
def datas = findDistrictData(city, cat, ["city","region", "name", "category", "location"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def dist = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def map = createData(dist, networks)
|
||||
if(map){
|
||||
rets << map
|
||||
}
|
||||
// println city +" "+cat +" "+totalCount.get() +"/"+datas.size()+" "+dist.name+" "+dist._id
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
saveDatas(rets);
|
||||
println cat +" "+city +" 总量:"+datas.size()+" 在5公里内总量:"+rets.size()
|
||||
}
|
||||
|
||||
def saveDatas(def rets){
|
||||
NETWORK_COLL.insert(rets)
|
||||
}
|
||||
|
||||
def createData(dist, networks){
|
||||
|
||||
def lastRest = null
|
||||
def minDistance = Integer.MAX_VALUE
|
||||
for(def row : networks){
|
||||
def locationstr = row['E']
|
||||
if(locationstr){
|
||||
def arrs = locationstr.split(",")
|
||||
def distance = GeoUtils.calculateDistance(dist.location.lon, dist.location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
|
||||
if(distance <= minDistance){
|
||||
minDistance = distance
|
||||
lastRest = row
|
||||
}
|
||||
}
|
||||
}
|
||||
if(minDistance<=5000){
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("_id", dist._id);
|
||||
obj.put("city", dist.city);
|
||||
obj.put("region", dist.region);
|
||||
obj.put("name", dist.name);
|
||||
obj.put("distance", (int)minDistance);
|
||||
obj.put("network", lastRest['B'])
|
||||
return obj
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def findDistrictData(def city, cat, fields){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status", 0);
|
||||
q.put("category", cat);
|
||||
q.put("city", city);
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
|
||||
return SEAWEED_CONN.find(q, keys).toList();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class PoiAndSeaweedExportCSV extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new PoiAndSeaweedExportCSV(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
|
||||
|
||||
poiExport(citys)
|
||||
|
||||
}
|
||||
|
||||
def poiExport(citys){
|
||||
def fileName = "/home/goojia/xiamen/BIGDATA_POI.csv"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\厦门\\BIGDATA_POI.csv"
|
||||
}
|
||||
|
||||
for(def c : citys){
|
||||
def rets = CommonDatautils.findNewAmapDetail(c)
|
||||
FileUtils.batchCreateFileBySplitChar(rets, fileName, null, "utf-8", true, "@|@");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,692 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xingyebank.Data
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PoiWash {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
|
||||
// DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
|
||||
|
||||
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
public static def townsMap = [:]
|
||||
public static def regions = []
|
||||
public static def nameKeys = null
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiWash()
|
||||
// task.updateId()
|
||||
// task.run()
|
||||
// task.synch()
|
||||
// task.check()
|
||||
task.deduplicate()
|
||||
// task.updateEsMapPoi()
|
||||
// task.checkSpecialSymbols()
|
||||
// task.splitRoad()
|
||||
}
|
||||
|
||||
def updateId(){
|
||||
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
for(def city : citys){
|
||||
def datas = findPois(city)
|
||||
update(datas)
|
||||
}
|
||||
}
|
||||
|
||||
def update(datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
|
||||
def count = datas.size()
|
||||
for(def i=0;i<datas.size();i++){
|
||||
def map = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("id", map._id.toString())
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+map.city+" 更改id"
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def synch(){
|
||||
def rets = []
|
||||
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
for(def city : citys){
|
||||
def datas = findAllFieldPois(city)
|
||||
println city +" "+datas.size()+" 同步"
|
||||
for(def d : datas){
|
||||
rets << d
|
||||
if(rets.size()%10000==0){
|
||||
MAPPOI_CONN2.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
MAPPOI_CONN2.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def checkSpecialSymbols(){
|
||||
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
|
||||
def citys = ["重庆市"]
|
||||
for(def city : citys){
|
||||
findAllFieldPois(city)
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def specialSymbols(city, results){
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def count = results.size()
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def columns = ["province","city","district","name","address","cat","business_area","fjw_province","fjw_city","type","secondary_type","small_type","township", "fjw_district","spell_name"]
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(columns.contains(k)){
|
||||
if(v instanceof Double){
|
||||
udata.put(k, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
if(udata){
|
||||
updateTotalCount.addAndGet(1)
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"
|
||||
//"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市"
|
||||
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
|
||||
for(def city : citys){
|
||||
townsMap = findAllTownship(city)
|
||||
regions = findAllRegions(city)
|
||||
def datas = findPois(city)
|
||||
println city +' poi: '+datas.size()+" towns:"+ townsMap.get(RegionData.formatCity(city)).size()
|
||||
process(city, datas)
|
||||
// updateOldRegion(city)
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def process(city, results){
|
||||
// def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def count = results.size()
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def newAddr = washing(map)
|
||||
if(map.address!=newAddr){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", newAddr)
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
// return rets;
|
||||
}
|
||||
|
||||
def washing(data){
|
||||
if(data.address){
|
||||
def addr = data.address.replace('[]', '').replace(' ','').replace(data.province,'')
|
||||
|
||||
if(data.province){
|
||||
addr = addr.replace(data.province,'')
|
||||
}
|
||||
if(data.city){
|
||||
addr = addr.replace(data.city,'')
|
||||
}
|
||||
if(data.district){
|
||||
if(!addr.contains(data.district+"路")){
|
||||
addr = addr.replace(data.district,'')
|
||||
}
|
||||
}
|
||||
if(data.township){
|
||||
try{
|
||||
addr = addr.replace(data.township,'')
|
||||
}catch(Exception e){
|
||||
println data.id+" "+data.township
|
||||
}
|
||||
|
||||
}
|
||||
if(addr && addr.startsWith('-')){
|
||||
addr = addr.replace('---','').replace('--','').replace('-','')
|
||||
}
|
||||
if(addr && addr.isNumber()){
|
||||
return null;
|
||||
}
|
||||
|
||||
if(addr){
|
||||
for(def r : regions){
|
||||
addr = addr.replace(r, "")
|
||||
}
|
||||
}
|
||||
|
||||
if(addr){
|
||||
def towns = townsMap.get(data.fjw_city)
|
||||
for(def t : towns){
|
||||
if(addr.contains(t)){
|
||||
addr = addr.replace(t, "")
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
return addr
|
||||
}
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("test", 1)
|
||||
// q.put("name", '(三星)企业设立登记第二受理点')
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("address", 1)
|
||||
keys.put("fjw_city", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()>=300000){
|
||||
print "."
|
||||
break;
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findAllFieldPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(10000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()>0 && rets.size()%100000==0){
|
||||
print "."
|
||||
specialSymbols(city, rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
specialSymbols(city, rets)
|
||||
rets.clear()
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def updateOldRegion(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("id", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("fjw_city", 1)
|
||||
keys.put("fjw_district", 1)
|
||||
keys.put("township", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000)
|
||||
def rets = []
|
||||
|
||||
def distMap = [:]
|
||||
for(def d : dbcursor){
|
||||
// def region = distMap.get(d.fjw_district)
|
||||
// if(!region){
|
||||
// distMap.put(d.fjw_district, d.district)
|
||||
// }
|
||||
//
|
||||
// if(d.district instanceof Double){
|
||||
// def newRegion = distMap.get(d.fjw_district)
|
||||
// if(newRegion){
|
||||
// println d.id +" "+d.fjw_district+" "+newRegion
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("district", newRegion)
|
||||
// MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
|
||||
if(d.township instanceof Double){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", "")
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
|
||||
println d.id
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllRegions(cityName){
|
||||
def q = new BasicDBObject()
|
||||
if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){
|
||||
q.put("province_name", cityName)
|
||||
}else{
|
||||
q.put("city_name", cityName)
|
||||
}
|
||||
def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000)
|
||||
def set = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
set << d.area_name
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
|
||||
def findAllTownship(cityName){
|
||||
def q = new BasicDBObject()
|
||||
if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){
|
||||
q.put("province_name", cityName)
|
||||
}else{
|
||||
q.put("city_name", cityName)
|
||||
}
|
||||
def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def city = d.fangjia_city
|
||||
if(!city || city instanceof Double){
|
||||
city = RegionData.formatCity(d.city_name)
|
||||
}
|
||||
def towns = map.get(city)
|
||||
if(!towns){
|
||||
towns = []
|
||||
map.put(city, towns)
|
||||
}
|
||||
def alias = d.alias
|
||||
towns << d.street_name
|
||||
if(alias){
|
||||
towns.addAll(alias)
|
||||
}
|
||||
// towns.unique()
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def deduplicate(){
|
||||
//"三明市","漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"
|
||||
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
|
||||
def total = 0
|
||||
def upindex = 0
|
||||
for(def city : citys){
|
||||
nameKeys = new HashSet()
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("status", new BasicDBObject('$ne', 2))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("id", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
total++
|
||||
def key = d.city +"##"+d.district+"##"+d.name+"##"+d.location.lon+"##"+d.location.lat
|
||||
if(!nameKeys.contains(key)){
|
||||
nameKeys << key
|
||||
}else{
|
||||
upindex++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 2)
|
||||
MAPPOI_CONN.update(new BasicDBObject('id',d.id),new BasicDBObject('$set', udata))
|
||||
println "[${upindex}/${total}] Delete : "+d.city +" "+ d.district+" "+d.name
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def updateEsMapPoi(){
|
||||
// "三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市",
|
||||
def citys = ["龙岩市","莆田市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
for(def city : citys) {
|
||||
def datas = findPoiByQuery(city)
|
||||
for(def d : datas){
|
||||
updateMapPoi(d.id)
|
||||
println city +" "+d.id
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def splitRoad(){
|
||||
//"三明市",
|
||||
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
|
||||
for(def city : citys) {
|
||||
def datas = findPoiByQuery(city, ['city','address'])
|
||||
splitRoadRun(datas)
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def splitRoadRun(results){
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def count = results.size()
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
|
||||
if(jsonObject && jsonObject.containsKey("result")){
|
||||
def dataMap = jsonObject.get("result")
|
||||
if(dataMap.get("street")){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street", dataMap.get("street"))
|
||||
if(dataMap.get("streetNumber")){
|
||||
udata.put("street_number", dataMap.get("streetNumber"))
|
||||
}
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/map_poi/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.status=2",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}
|
||||
|
||||
|
||||
def findPoiByQuery(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("status", 2)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("id", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for (def d : dbcursor) {
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findPoiByQuery(city, fields){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
fields.each{
|
||||
keys.put(it, 1)
|
||||
}
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(5000)
|
||||
def rets = []
|
||||
for (def d : dbcursor) {
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def check(){
|
||||
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
|
||||
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
|
||||
for(def city : citys){
|
||||
def stop = false
|
||||
while(!stop){
|
||||
def datas = findPois(city)
|
||||
if(datas.size()>0){
|
||||
processUpdate(city, datas)
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def processUpdate(city, results){
|
||||
// def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
def updateTotalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def count = results.size()
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
def cat = map.cat
|
||||
if(cat && cat.contains(":")){
|
||||
def arrs = map.cat.split(":")
|
||||
udata.put("type",arrs[0])
|
||||
if(arrs.size()>=2){
|
||||
udata.put("secondary_type",arrs[1])
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
udata.put("small_type",arrs[2])
|
||||
}
|
||||
if(arrs.size()>3){
|
||||
udata.put("cat",arrs[0]+";"+arrs[1]+";"+arrs[2])
|
||||
}
|
||||
}
|
||||
|
||||
if(map.bd_location){
|
||||
def new_bd_lon = DataHandleUtils.getDoubleRound(6, map.bd_location.lon)
|
||||
def new_bd_lat = DataHandleUtils.getDoubleRound(6, map.bd_location.lat)
|
||||
def new_bd_map = ["lon":new_bd_lon, "lat":new_bd_lat]
|
||||
udata.put("bd_location", new_bd_map)
|
||||
}
|
||||
|
||||
if(map.gps_location){
|
||||
def new_gps_lon = DataHandleUtils.getDoubleRound(6, map.gps_location.lon)
|
||||
def new_gps_lat = DataHandleUtils.getDoubleRound(6, map.gps_location.lat)
|
||||
def new_gps_map = ["lon":new_gps_lon, "lat":new_gps_lat]
|
||||
udata.put("gps_location", new_gps_map)
|
||||
}
|
||||
if(map.location){
|
||||
def new_lon = DataHandleUtils.getDoubleRound(6, map.location.lon)
|
||||
def new_lat = DataHandleUtils.getDoubleRound(6, map.location.lat)
|
||||
def new_map = ["lon":new_lon, "lat":new_lat]
|
||||
udata.put("location", new_map)
|
||||
}
|
||||
|
||||
|
||||
|
||||
if(map.address && (map.address.contains("(") || map.address.contains("("))){
|
||||
def s1 = null
|
||||
def addr = map.address.replace("(", "(").replace(")", ")")
|
||||
if(addr.contains("(") && addr.contains(")")){
|
||||
s1 = addr.substring(addr.indexOf("("),addr.indexOf(")")+1 )
|
||||
}
|
||||
|
||||
if(s1){
|
||||
def new_addr = addr.replace(s1, "")
|
||||
if(DataHandleUtils.isStandardRoad(new_addr)){
|
||||
udata.put("address", new_addr)
|
||||
}else{
|
||||
if(DataHandleUtils.isStandardRoad(s1)){
|
||||
udata.put("address", s1.replace("(", "").replace(")", ""))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(udata){
|
||||
updateTotalCount.addAndGet(1)
|
||||
// udata.put("test", 2)
|
||||
MAPPOI_CONN.update(new BasicDBObject('id',map.id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
}
|
||||
|
||||
|
||||
// if(map.address){
|
||||
// def newAddr = DataUtil.toDBCcase(map.address)
|
||||
//
|
||||
// if(newAddr.startsWith(";")){
|
||||
// newAddr = newAddr.substring(1)
|
||||
// }
|
||||
// if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){
|
||||
// newAddr = ""
|
||||
// }
|
||||
//
|
||||
// if(map.address!=newAddr){
|
||||
// updateTotalCount.addAndGet(1)
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("address", newAddr)
|
||||
// MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
|
||||
// println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
// }
|
||||
// }
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
latch.await(3, TimeUnit.MINUTES);
|
||||
// return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class PostCodeSynchBatch extends BaseBatch{
|
||||
|
||||
public static final DBCollection post_code_coll = MongoUtils.getCollectionByName("crawl", "post_code", "192.168.0.206", 27020)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection POST_CODE_CONN = MongoUtils.getCollectionByName2("fangjia", "postcode", MONGO100_URL)
|
||||
|
||||
def fullCityMap = null
|
||||
def mcaDistMap = null
|
||||
|
||||
static main(args) {
|
||||
runBatch(new PostCodeSynchBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCity()
|
||||
|
||||
def datas = findAllDatas();
|
||||
for(def d : datas){
|
||||
def map = [:]
|
||||
map._id = d._id
|
||||
def area_code = null
|
||||
map.province = d.province
|
||||
|
||||
|
||||
def short_city = RegionData.formatCity(d.city)
|
||||
if(short_city){
|
||||
def data = fullCityMap.get(short_city)
|
||||
if(data){
|
||||
map.city = data.city
|
||||
map.city_code = data.mca_city_code
|
||||
map.province_code = data.mca_province_code
|
||||
}
|
||||
|
||||
def region = d.district.replace(d.province, "").replace(d.city, "")
|
||||
def short_district = RegionData.formatRegion(short_city, region)
|
||||
|
||||
def distData = mcaDistMap.get(short_city+"#"+short_district)
|
||||
if(distData){
|
||||
map.district = distData.district
|
||||
map.district_code = distData.mca_district_code
|
||||
}else{
|
||||
println d.city +" "+d.district+" 民政部中没有找到相应的区域全称"
|
||||
}
|
||||
|
||||
}else{
|
||||
println d.city +" 没有标准化出来"
|
||||
}
|
||||
map.city_zip_code = d.city_code
|
||||
map.district_zip_code = d.district_code
|
||||
map.address = d.addr
|
||||
map.addr_zip_code = d.addr_code
|
||||
|
||||
updateData(map)
|
||||
}
|
||||
}
|
||||
|
||||
def updateData(map){
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(k!="_id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
POST_CODE_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = post_code_coll.find(q)
|
||||
return dbcursor.toList();
|
||||
}
|
||||
}
|
||||
+369
@@ -0,0 +1,369 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
|
||||
class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
|
||||
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
|
||||
|
||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||
|
||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
public static def priceMap = null
|
||||
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
|
||||
|
||||
def fullCityMap = null
|
||||
def mcaDistMap = null
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SeaweedDataSynch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
|
||||
run()
|
||||
// println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
//codeRun()
|
||||
//println "民政部省,市,区同步完成"
|
||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
|
||||
// addTest()
|
||||
}
|
||||
|
||||
def addTest(){
|
||||
def q = new BasicDBObject()
|
||||
def ret = []
|
||||
def dbcurosr = community_100_CONN.find(q).batchSize(20000)
|
||||
for(def d : dbcurosr){
|
||||
def m = new BasicDBObject()
|
||||
m.put("province" , d.province_name)
|
||||
m.put("province_code", d.province_code)
|
||||
if(['北京市','上海市','重庆市','天津市'].contains(d.province_name)){
|
||||
m.put("city" , d.province_name)
|
||||
}else if(d.area_name=="自治区直辖县级行政区划" || d.area_name=="省直辖县级行政区划"){
|
||||
m.put("city" , d.area_name)
|
||||
}else{
|
||||
m.put("city" , d.city_name)
|
||||
}
|
||||
m.put("city_code" ,d.city_code)
|
||||
m.put("district" , d.area_name)
|
||||
m.put("district_code" , d.area_code)
|
||||
m.put("town" , d.street_name)
|
||||
m.put("town_code" , d.street_code)
|
||||
m.put("committee" , d.name)
|
||||
m.put("committee_code" , d.code)
|
||||
m.put("location",d.location)
|
||||
m.put("type_code" ,d.type)
|
||||
if(d.type=="111"){
|
||||
m.put("type" , "主城区")
|
||||
}else if(d.type=="112"){
|
||||
m.put("type" , "城乡结合区")
|
||||
}else if(d.type=="121"){
|
||||
m.put("type" , "镇中心区")
|
||||
}else if(d.type=="122"){
|
||||
m.put("type" , "镇乡结合区")
|
||||
}else if(d.type=="123"){
|
||||
m.put("type" , "特殊区域")
|
||||
}else if(d.type=="210"){
|
||||
m.put("type" , "乡中心区")
|
||||
}else if(d.type=="220"){
|
||||
m.put("type" , "村庄")
|
||||
}
|
||||
|
||||
ret << m
|
||||
|
||||
if(ret.size()%50000==0){
|
||||
stat_administrative_division_100_CONN.insert(ret)
|
||||
ret = []
|
||||
}
|
||||
}
|
||||
if(ret.size()>0){
|
||||
stat_administrative_division_100_CONN.insert(ret)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCity()
|
||||
|
||||
// priceMap = findAllDistrictSellPrice()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index++
|
||||
updating(c, "district")
|
||||
updating(c, "office")
|
||||
updating(c, "shop")
|
||||
println index +" "+ c + " 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def map = createData(dist)
|
||||
|
||||
//往100 上同步
|
||||
updateData(map)
|
||||
|
||||
//往12 es上同步
|
||||
// def newUrl = targetUrl +"_doc/"+map.id
|
||||
// CommonDatautils.httpSaveData(newUrl, map)
|
||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def updateData(map){
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(k!="_id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
SEAWEED_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
|
||||
def createData(dist){
|
||||
def map = [:]
|
||||
map._id = dist._id
|
||||
def area_code = null
|
||||
def province = provinceMap.get(dist.city)
|
||||
if(province){
|
||||
map.province = ProvinceData.getFullProvince(province)
|
||||
}
|
||||
map.fangjia_province = province
|
||||
|
||||
def data = fullCityMap.get(dist.city)
|
||||
if(data){
|
||||
map.city = data.city
|
||||
area_code = data.mca_city_code
|
||||
}
|
||||
map.fangjia_city = dist.city
|
||||
|
||||
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
|
||||
if(distData){
|
||||
map.region = distData.district
|
||||
area_code = distData.mca_district_code
|
||||
}else{
|
||||
// map.region = dist.region
|
||||
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
|
||||
}
|
||||
map.area_code = area_code
|
||||
|
||||
map.fangjia_region = dist.region
|
||||
|
||||
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
map.block = dist.block
|
||||
if(dist.mapping_alias){
|
||||
def aliasNames = []
|
||||
for(def key : dist.mapping_alias){
|
||||
if(key && !key.toString().equals("null")){
|
||||
aliasNames << key
|
||||
if(aliasNames.size()>5){
|
||||
map.alias = aliasNames
|
||||
break;
|
||||
}else{
|
||||
if(aliasNames){
|
||||
map.alias = aliasNames
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(dist.addr_alias){
|
||||
def names = []
|
||||
for(def key : dist.addr_alias){
|
||||
if(key && !key.toString().equals("null")){
|
||||
names << key
|
||||
if(names.size()>5){
|
||||
map.addr_alias = names
|
||||
break;
|
||||
}else{
|
||||
if(names){
|
||||
map.addr_alias = names
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
map.category = dist.cat
|
||||
map.estate_type = dist.estate_type2
|
||||
map.status = dist.status
|
||||
map.township = dist.township
|
||||
map.finish_build_time = dist.create_year?dist.create_year.toString():null
|
||||
if(dist.lng){
|
||||
def loc = [:]
|
||||
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
|
||||
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
|
||||
map.location = loc
|
||||
}
|
||||
|
||||
if(map.location){
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
}
|
||||
|
||||
map.build_type = dist.build_type
|
||||
map.households = dist.household_count
|
||||
map.committee = dist.committee
|
||||
if(dist.estate_charge && dist.estate_charge.toString().isNumber()){
|
||||
def fee = dist.estate_charge.toString().toFloat()
|
||||
if(fee < 100){
|
||||
map.property_fee = DataHandleUtils.getDoubleRound(2, fee)
|
||||
}
|
||||
}
|
||||
|
||||
if(dist.car_site_count && dist.car_site_count.toString().isNumber()){
|
||||
map.car_site_count = dist.car_site_count.toString().toInteger()
|
||||
}
|
||||
if(dist.property_life && dist.property_life.toString().isNumber()){
|
||||
map.property_life = dist.property_life.toString().toInteger()
|
||||
}
|
||||
|
||||
map.house_property = dist.house_property
|
||||
|
||||
if(dist.build_area && dist.build_area.toString().isNumber()){
|
||||
map.build_area = dist.build_area.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
if(dist.all_area && dist.all_area.toString().isNumber()){
|
||||
map.floor_area = dist.all_area.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
map.build_structure = dist.build_structure
|
||||
map.developer = dist.developer
|
||||
map.property_company = dist.estate_company
|
||||
map.plot_ratio = dist.cubage_percent
|
||||
map.landscaping_ratio = dist.virescence_percent
|
||||
if(dist.loudong_count && dist.loudong_count.toString().isNumber()){
|
||||
map.building_count = dist.loudong_count.toString().toInteger()
|
||||
}
|
||||
|
||||
map.schools = dist.schools
|
||||
map.main_house_type = dist.main_type
|
||||
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
|
||||
map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat())
|
||||
}
|
||||
if(priceMap){
|
||||
def dataMap = priceMap.get(dist._id.toString())
|
||||
if(dataMap){
|
||||
map.avg_price = dataMap.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
return map
|
||||
}
|
||||
def findAllDistrictSellPrice(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("date", lastMonthDate)
|
||||
def priceMap = [:]
|
||||
DISTRICT_SELL_STAT.find(q).each{
|
||||
priceMap.put(it.commid, it)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
def codeRun(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = mca_administrative_division_coll.find(q)
|
||||
def rets = []
|
||||
for(def it : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("_id", it._id)
|
||||
udata.put("province", it.province)
|
||||
udata.put("city", it.city)
|
||||
udata.put("district", it.district)
|
||||
udata.put("mca_province_code", it.province_code)
|
||||
udata.put("mca_city_code", it.city_code)
|
||||
udata.put("mca_district_code", it.district_code)
|
||||
udata.put("level", it.level)
|
||||
udata.put("c_date", it.c_date)
|
||||
rets << udata
|
||||
// administrative_division_100_CONN.save(udata)
|
||||
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
administrative_division_100_CONN.insert(rets)
|
||||
}
|
||||
}
|
||||
+461
@@ -0,0 +1,461 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.lixue.DB.City
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.*
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
|
||||
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
|
||||
|
||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||
|
||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
public static def priceMap = null
|
||||
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
|
||||
def keys = new CopyOnWriteArraySet();
|
||||
def fullCityMap = null
|
||||
def mcaDistMap = null
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SuggestionDataSynch(), args);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
run()
|
||||
// updateCode()
|
||||
// getExitsCity()
|
||||
|
||||
}
|
||||
|
||||
def getExitsCity(){
|
||||
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
|
||||
def cities = SUGGESTION_CONN.distinct("city", new BasicDBObject("type","poi"))
|
||||
def simpleCities = []
|
||||
for (def c :cities) {
|
||||
def city = RegionData.formatCity(c)
|
||||
simpleCities.add(city)
|
||||
}
|
||||
// print(simpleCities)
|
||||
return simpleCities
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def run() {
|
||||
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
fullCityMap = queryFullNameCity()
|
||||
mcaDistMap = queryDistNameCity()
|
||||
def existCities = getExitsCity()
|
||||
|
||||
for (def c : citys) {
|
||||
|
||||
def allRets = []
|
||||
|
||||
// if(["上海","北京","广州","深圳","成都","杭州","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳","怀化","郴州","益阳","永州","娄底","湘西","长沙","常德","株洲","岳阳","衡阳","湘潭","张家界","邵阳","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁"].contains(c)){
|
||||
if(existCities.contains(c)){
|
||||
continue;
|
||||
}
|
||||
index++
|
||||
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
||||
def ret1 = updating(c, "district")
|
||||
def ret2 = updating(c, "office")
|
||||
def ret3 = updating(c, "shop")
|
||||
|
||||
// }
|
||||
// if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
|
||||
def ret4 = poiData(c)
|
||||
// }
|
||||
|
||||
allRets.addAll(ret1)
|
||||
allRets.addAll(ret2)
|
||||
allRets.addAll(ret3)
|
||||
allRets.addAll(ret4)
|
||||
|
||||
saveToMG(c, allRets)
|
||||
println index + " " + c + " 完成 "+ allRets.size()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def saveToMG(city, rows){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
def q = new BasicDBObject("city", data.city)
|
||||
def count = 0
|
||||
try{
|
||||
count = SUGGESTION_CONN.count(q)
|
||||
}catch(e){
|
||||
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
|
||||
count = SUGGESTION_CONN.count(q)
|
||||
}
|
||||
// if(count>0){
|
||||
// SUGGESTION_CONN.remove(q)
|
||||
// }
|
||||
|
||||
def rets = []
|
||||
for(def i=0;i<rows.size();i++){
|
||||
def m = rows[i]
|
||||
def obj = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
rets << obj
|
||||
if(i%50000==0){
|
||||
SUGGESTION_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
SUGGESTION_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
def key = dist.city+"#"+dist.region+"#"+dist.name
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
|
||||
def map = createData(dist)
|
||||
//往100 上同步
|
||||
// updateData(map)
|
||||
rets << map
|
||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def createData(dist){
|
||||
def map = [:]
|
||||
map.id = dist._id.toString()
|
||||
def province = provinceMap.get(dist.city)
|
||||
if(province){
|
||||
map.province = ProvinceData.getFullProvince(province)
|
||||
}
|
||||
|
||||
def data = fullCityMap.get(dist.city)
|
||||
map.city = data.city
|
||||
|
||||
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
|
||||
if(distData){
|
||||
map.region = distData.district
|
||||
}else{
|
||||
map.region = dist.region
|
||||
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
|
||||
}
|
||||
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
if(dist.alias && dist.alias.size()>0){
|
||||
map.alias = dist.alias
|
||||
}
|
||||
if(dist.addr_alias && dist.addr_alias.size()>0){
|
||||
map.addr_alias = dist.addr_alias
|
||||
}
|
||||
map.type = dist.cat
|
||||
if(dist.lng){
|
||||
def loc = [:]
|
||||
loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0)
|
||||
loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0)
|
||||
map.location = loc
|
||||
}
|
||||
map.source = "FJW"
|
||||
map.township = dist.township
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findAllDistrictSellPrice(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("date", lastMonthDate)
|
||||
def priceMap = [:]
|
||||
DISTRICT_SELL_STAT.find(q).each{
|
||||
priceMap.put(it.commid, it)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
|
||||
def updateData(map){
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(k!="id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
SUGGESTION_CONN.update(new BasicDBObject('_id', map.id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
|
||||
|
||||
def poiData(city){
|
||||
def townMap = CommonDatautils.findTownRange(city)
|
||||
def data = fullCityMap.get(city)
|
||||
if(data){
|
||||
def results = CommonDatautils.findAmapDetailBySql(data.city)
|
||||
def rets = createPoi(data.city, results, data, townMap)
|
||||
return rets;
|
||||
}
|
||||
return []
|
||||
|
||||
}
|
||||
|
||||
|
||||
def createPoi(fullCity, results, data, townMap){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
def townshipMap = findMapPoiTownshipMap(fullCity)
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
for(def i=0;i<results.size();i++){
|
||||
final def dist = results[i]
|
||||
def key = dist.city+"#"+dist.district+"#"+dist.name
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
|
||||
def map = [:]
|
||||
map.id = dist.id
|
||||
map.province = dist.province
|
||||
map.city = dist.city
|
||||
map.region = dist.district
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
map.type = "poi"
|
||||
map.cat = dist.cat
|
||||
map.location = dist.location
|
||||
map.source = "FJW"
|
||||
|
||||
if(townshipMap && townshipMap.containsKey(dist.id)){
|
||||
def township = townshipMap.get(dist.id)
|
||||
if(township){
|
||||
map.township = township
|
||||
}
|
||||
}
|
||||
if(!map.township){
|
||||
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
|
||||
if(!map.township){
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
}
|
||||
}
|
||||
println map.township
|
||||
//往100 上同步
|
||||
// updateData(map)
|
||||
rets << map
|
||||
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findMapPoiTownshipMap(fullCity){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", fullCity)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("id", 1)
|
||||
keys.put("township", 1)
|
||||
def dbcursor = MAPPOI_CONN.find(q, keys)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
map.put(d.id, d.township)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findTownshipByBD(id){
|
||||
def q = new BasicDBObject()
|
||||
q.put("id", id)
|
||||
def obj = MAPPOI_CONN.findOne(q)
|
||||
return obj
|
||||
|
||||
}
|
||||
|
||||
|
||||
def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def fjwcity = null
|
||||
if(d.level == "城市"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(!fjwcity){
|
||||
if(d.city == "市辖区"){
|
||||
fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
d.city = d.province
|
||||
}
|
||||
}
|
||||
}
|
||||
}else if(d.level == "区域"){
|
||||
if(d.administer == "省直辖县级行政区划"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
d.city = d.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(fjwcity){
|
||||
retMap.put(fjwcity, d)
|
||||
}else{
|
||||
println d.toString() +" 格式化不成功!"
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("level","区域")
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.city == "省直辖县级行政区划"){
|
||||
continue
|
||||
}else if(d.city == "市辖区"){
|
||||
def fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}else{
|
||||
def fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
def updateCode(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
for(def d : dbcursor){
|
||||
def code = null
|
||||
if(d.mca_district_code){
|
||||
code = d.mca_district_code
|
||||
}else if(d.mca_city_code){
|
||||
code = d.mca_city_code
|
||||
}else if(d.mca_province_code){
|
||||
code = d.mca_province_code
|
||||
}
|
||||
|
||||
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("code", code);
|
||||
administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata));
|
||||
}
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.district = district
|
||||
map.township = township
|
||||
return map;
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -45,10 +45,10 @@ class PdsUseStatisticBatch extends BaseBatch{
|
||||
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
|
||||
def static emails = [
|
||||
|
||||
"duweiliang@fangjia.com",
|
||||
"yaojingmeng@fangjia.com",
|
||||
"xuqi@fangjia.com",
|
||||
"wangpeili@fangjia.com"
|
||||
// "duweiliang@fangjia.com",
|
||||
// "yaojingmeng@fangjia.com",
|
||||
// "xuqi@fangjia.com",
|
||||
"gaojuan@fangjia.com"
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
|
||||
+2
-2
@@ -163,9 +163,9 @@ class AnalyticsApiLogPerformanceBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
static void sendEmail(String sub, String msg) {
|
||||
jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
|
||||
// jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
|
||||
jms.sendMail("wangwenyan@fangjia.com", sub, msg)
|
||||
jms.sendMail("helvyuan@fangjia.com", sub, msg)
|
||||
// jms.sendMail("helvyuan@fangjia.com", sub, msg)
|
||||
jms.sendMail("gaojuan@fangjia.com", sub, msg)
|
||||
}
|
||||
|
||||
|
||||
+5
-5
@@ -32,11 +32,11 @@ class CommonLogAnaBatch extends BaseBatch {
|
||||
def static bjEvaReport = [];
|
||||
|
||||
public static final def EMAILS = [
|
||||
"jiangxiaochun@fangjia.com",
|
||||
"limingmin@fangjia.com",
|
||||
"xiaohai@fangjia.com",
|
||||
"luxuping@fangjia.com",
|
||||
"yukai@fangjia.com"
|
||||
"jiangxiaochun@fangjia.com"
|
||||
// "limingmin@fangjia.com",
|
||||
// "xiaohai@fangjia.com",
|
||||
// "luxuping@fangjia.com",
|
||||
// "yukai@fangjia.com"
|
||||
];
|
||||
|
||||
static main(args) {
|
||||
|
||||
+6
-6
@@ -26,12 +26,12 @@ class DaikuanLogAnaBatch extends BaseBatch {
|
||||
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
|
||||
|
||||
public static final def EMAILS = [
|
||||
"jiangxiaochun@fangjia.com",
|
||||
"limingmin@fangjia.com",
|
||||
"zhongjingyi@fangjia.com",
|
||||
"xiaohai@fangjia.com",
|
||||
"luxuping@fangjia.com",
|
||||
"yukai@fangjia.com"
|
||||
"jiangxiaochun@fangjia.com"
|
||||
// "limingmin@fangjia.com",
|
||||
// "zhongjingyi@fangjia.com",
|
||||
// "xiaohai@fangjia.com",
|
||||
// "luxuping@fangjia.com",
|
||||
// "yukai@fangjia.com"
|
||||
];
|
||||
|
||||
static main(args) {
|
||||
|
||||
+7
-7
@@ -148,14 +148,14 @@ class ReadBkoffSafeLogBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
static void sendEmail(String sub, String msg) {
|
||||
jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
|
||||
jms.sendMail("joe@fangjia.com", sub, msg)
|
||||
jms.sendMail("xiaohai@fangjia.com", sub, msg)
|
||||
// jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
|
||||
// jms.sendMail("joe@fangjia.com", sub, msg)
|
||||
// jms.sendMail("xiaohai@fangjia.com", sub, msg)
|
||||
jms.sendMail("caoyuyan@fangjia.com", sub, msg)
|
||||
jms.sendMail("wangchencen@fangjia.com", sub, msg)
|
||||
jms.sendMail("yaojingmeng@fangjia.com", sub, msg)
|
||||
jms.sendMail("chichilook@qq.com", sub, msg)
|
||||
jms.sendMail("sunjian@fangjia.com", sub, msg)
|
||||
// jms.sendMail("wangchencen@fangjia.com", sub, msg)
|
||||
// jms.sendMail("yaojingmeng@fangjia.com", sub, msg)
|
||||
// jms.sendMail("chichilook@qq.com", sub, msg)
|
||||
// jms.sendMail("sunjian@fangjia.com", sub, msg)
|
||||
}
|
||||
|
||||
static public String getCatById(String id) {
|
||||
|
||||
+21
@@ -272,4 +272,25 @@ class InsidePolygon {
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
public static def isInArea2(lng, lat, pointMap){
|
||||
def flag = false;
|
||||
Set<String> keys = pointMap.keySet();
|
||||
def name = null
|
||||
for(String key :keys){
|
||||
def prints = pointMap.get(key)
|
||||
for(def pointArr : prints){
|
||||
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
|
||||
if(flag){
|
||||
def arrs = key.split("##")
|
||||
name = arrs[1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(flag){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package com.fangjia.script.majintao
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
|
||||
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
def map = [
|
||||
["学校", "高等院校", "中学", "小学", "幼儿园", "成人教育", "职业技术学校"]: "学校",
|
||||
["综合医院", "专科医院"] : "医院",
|
||||
["银行"] : "银行",
|
||||
["超级市场", "家电电子卖场", "便民商店/便利店"] : "超市",
|
||||
["商场"] : "商场",
|
||||
["美容美发店"] : "美容院",
|
||||
["健身中心"] : "健身中心",
|
||||
["公园广场", "公园"] : "公园/绿地",
|
||||
["公交车站"] : "公交",
|
||||
["飞机场"] : "机场",
|
||||
["火车站"] : "火车站",
|
||||
["快餐厅", "外国餐厅", "中餐厅"] : "餐馆",
|
||||
["宾馆酒店", "旅馆招待所"] : "酒店"
|
||||
] as HashMap<List<String>, String>
|
||||
|
||||
def type3In = ["学校", "高等院校", "中学", "小学", "幼儿园", "成人教育", "职业技术学校", "健身中心", "公园广场", "公园", "地铁站", "飞机场", "火车站"]
|
||||
// 输出 Map 的内容
|
||||
map.each { key, value ->
|
||||
println("$key: $value")
|
||||
}
|
||||
|
||||
def provinceList = []
|
||||
def provinceMap = ProvinceData.data
|
||||
provinceMap.each { pro ->
|
||||
provinceList.add(pro.getValue()[0])
|
||||
}
|
||||
def cityMap = ProvinceData.GOVERNMENT_CITY_LIST
|
||||
cityMap.each { city ->
|
||||
provinceList.add(city.getValue()[0])
|
||||
}
|
||||
def specialMap = ProvinceData.SPECIAL_ADMIN_REGION
|
||||
specialMap.each { spe ->
|
||||
provinceList.add(spe.getValue()[1])
|
||||
}
|
||||
|
||||
def pcmap = [:]
|
||||
provinceList.remove("广西省")
|
||||
provinceList.add("广西壮族自治区")
|
||||
|
||||
DSCollection seabird = new DataStore("192.168.0.43", 28017).getCollection("fangjia_base.baidu_place_poi2")
|
||||
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "192.168.0.43", 28017)
|
||||
//def type3Not = ["学校内部设施", "进港", "出港", "候船室", "住宿服务相关"]
|
||||
//
|
||||
//def type2Not = ["学校", "专科医院", "综合医院", "诊所", "银行", "自动提款机", "便民商店/便利店", "家电电子卖场", "超级市场", "商场", "美容美发店", "地铁站", "港口码头", "住宿服务"]
|
||||
//def type2In = ["运动场馆", "公园广场", "公交车站", "机场相关", "火车站", "餐饮服务"]
|
||||
// 运动场馆 且为 健身中心
|
||||
|
||||
// 公园广场 公园广场
|
||||
//公园广场 公园
|
||||
// 公交车站 公交车站相关
|
||||
|
||||
// 机场相关 飞机场
|
||||
// 火车站 火车站
|
||||
// 餐饮服务 快餐厅 外国餐厅 中餐厅
|
||||
|
||||
println(provinceList)
|
||||
//provinceList = provinceList.subList(0,1)
|
||||
def c = 0
|
||||
def citys = []
|
||||
def citys2 = []
|
||||
|
||||
|
||||
for (p in provinceList) {
|
||||
println(p)
|
||||
def result = []
|
||||
ElasticUtils.executeSqlQuery("SELECT location,name,adname,cityname,m_date,id,address,type FROM amap_poi WHERE pname = '" + p + "'", { Map data ->
|
||||
// println(data["type"] + "-----")
|
||||
c = c+1
|
||||
if (c%10000==0){
|
||||
println("已处理$c 条")
|
||||
}
|
||||
def locationString = String.valueOf(data.get("location"))
|
||||
def typeString = String.valueOf(data.get("type"))
|
||||
def cityString = String.valueOf(data.get("cityname"))
|
||||
// citys2.add(cityS)
|
||||
// print(cityString)
|
||||
String city
|
||||
if ("台湾省".equals(cityString) | "香港特别行政区".equals(cityString) | "澳门特别行政区".equals(cityString)) {
|
||||
city = cityString
|
||||
} else {
|
||||
city = RegionData.formatCity(cityString)
|
||||
}
|
||||
def m_date = data["m_date"]
|
||||
def region = data["adname"]
|
||||
region = RegionData.formatRegion(city,region)
|
||||
def address = String.valueOf(data.get("address"))
|
||||
def uid = String.valueOf(data.get("id"))
|
||||
|
||||
def name = data.get("name")
|
||||
if (typeString.contains("|")) {
|
||||
typeString = typeString.split("'|'")[0]
|
||||
}
|
||||
// println(typeString)
|
||||
def secondType = typeString.split(";")[1]
|
||||
def smallType = typeString.split(";")[2]
|
||||
// println(smallType.getClass().getName())
|
||||
|
||||
|
||||
// 使用正则表达式匹配经度和纬度
|
||||
def matcher = locationString =~ /(-?\d+\.\d+)/
|
||||
def keyword = ""
|
||||
def flag = 0
|
||||
// 类型映射
|
||||
// 用三级分类判断
|
||||
if (smallType in type3In) {
|
||||
// 遍历Map的键集合,检查目标字符串是否存在于键集合中
|
||||
map.keySet().each { key ->
|
||||
if (key.contains(smallType)) {
|
||||
def value = map[key]
|
||||
// println("$smallType 存在于Map的键中,对应的值为: $value")
|
||||
keyword = value
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
//用二级分类判断
|
||||
map.keySet().each { key ->
|
||||
if (key.contains(secondType)) {
|
||||
def value = map[key]
|
||||
// println("$secondType 存在于Map的键中,对应的值为: $value")
|
||||
keyword = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 获取匹配到的经度和纬度值
|
||||
def lng = matcher[0][0]
|
||||
lng = lng.toDouble()
|
||||
def lat = matcher[1][0]
|
||||
lat = lat.toDouble()
|
||||
|
||||
// println "纬度: $lng, 经度: $lat,"
|
||||
if (!"".equals(keyword)) {
|
||||
def dbo = new BasicDBObject()
|
||||
dbo["_id"] = new ObjectId()
|
||||
dbo["keyword"] = keyword
|
||||
dbo["lat"] = lat
|
||||
dbo["name"] = name
|
||||
dbo["city"] = city
|
||||
dbo["m_date"] = m_date
|
||||
dbo["lng"] = lng
|
||||
dbo["c_date"] = m_date
|
||||
dbo["address"] = address
|
||||
dbo["uid"] = uid
|
||||
dbo["status"] = 0
|
||||
dbo["source"] = "gaode"
|
||||
dbo["region"] = region
|
||||
result.add(dbo)
|
||||
}
|
||||
// print(result)
|
||||
// println("888")
|
||||
|
||||
})
|
||||
if (null != result && result.size() > 0) {
|
||||
m43.insert(result)
|
||||
}
|
||||
|
||||
println("已处理完$p 的数据")
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
package com.fangjia.script.majintao
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
def count = 0
|
||||
//ElasticUtils.executeSqlQuery("SELECT * FROM seawater WHERE city = '重庆' and region = '渝北' and district_name = '棕榈泉国际花园四期'and p_date > 20230801 ",{Map doc->
|
||||
ElasticUtils.executeSqlQuery("SELECT * FROM seawater WHERE p_date > 20230801 order by p_date desc limit 100",{Map doc->
|
||||
def keys = ["c_date","_id","cat","city","region","name","address","block","district_name","total_price","area","height","floor","fitment","estate_type","hall","toilet","publish_date","url","direction","build_type","pay_type","line","rent_type","status","room"]
|
||||
// def dbo = new BasicDBObject()
|
||||
// for( String key in keys){
|
||||
// dbo[key] = doc[key]
|
||||
// }
|
||||
// println(doc['m_date'])
|
||||
// print(doc['name'])
|
||||
if (count==0)print(doc)
|
||||
count+=1
|
||||
|
||||
// DbUtils.seabird.save(dbo)
|
||||
})
|
||||
//DbUtils.seabird.find(new BasicDBObject("district_name","棕榈泉国际花园四期")).each {
|
||||
// println(it)
|
||||
//}
|
||||
print(count)
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.lowagie.text.Row
|
||||
import javafx.scene.control.Cell
|
||||
import jxl.Sheet
|
||||
import jxl.Workbook
|
||||
import org.apache.poi.ss.usermodel.WorkbookFactory
|
||||
|
||||
//String excelFilePath = "F:\\估家\\文件\\安徽省联社\\全国省级行政区房屋覆盖情况表 - 副本.xlsx"
|
||||
//ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
//ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
//
|
||||
// province1 = items['A']
|
||||
// if (!province1.equals('黑龙江省') || !province1.equals('内蒙古自治区')) {
|
||||
// province = province1.substring(0, 2)
|
||||
// } else {
|
||||
// province = province1.substring(0, 3)
|
||||
// }
|
||||
//
|
||||
// println('province: ' + province)
|
||||
//
|
||||
// def citys = ProvinceData.getAllCity(province)
|
||||
// println(citys)
|
||||
//// if (citys.size() > 0) {
|
||||
//// ExcelUtils.writeToExcel(excelFilePath, citys, province)
|
||||
//// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
//
|
||||
//import com.fangjia.core.ProvinceData
|
||||
//
|
||||
//import com.fangjia.script.util.ExcelUtils
|
||||
//
|
||||
//
|
||||
//String excelFilePath = "F:\\估家\\文件\\安徽省联社\\全国省级行政区房屋覆盖情况表.xlsx"
|
||||
//ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
//ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
//
|
||||
// def province1 = items['A']
|
||||
// def province
|
||||
//
|
||||
// if (!province1.equals('黑龙江省') && !province1.equals('内蒙古自治区')) {
|
||||
// province = province1.substring(0, 2)
|
||||
// } else {
|
||||
// province = province1.substring(0, 3)
|
||||
// }
|
||||
//
|
||||
// println('province: ' + province)
|
||||
//
|
||||
// def citys = ProvinceData.getAllCity(province)
|
||||
// println(citys)
|
||||
// items['D'] = citys.toString()
|
||||
//
|
||||
//})
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.fangjia.script.majintao
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
|
||||
String excelFilePath = "F:\\估家\\文件\\北交所\\testcopy1.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
String comprehensiveUrl = "http://localhost:9180/comprehensiveQuery/houseValueInformation";
|
||||
String communityUrl = "http://open.fangjia.com/addrSvc/avgPrice";
|
||||
String valueUrl = "http://open.fangjia.com/property/addressEvaluate";
|
||||
String caseUrl = "http://open.fangjia.com/house/case";
|
||||
String poiUrl = "http://open.fangjia.com/district/addrFacilitys";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
String city = items['A']
|
||||
String address = items['B']
|
||||
double size = items['C']
|
||||
|
||||
System.out.println(city + "---" + address + "----" + size);
|
||||
|
||||
map.put("city", city)
|
||||
map.put("address", address)
|
||||
map.put("size", size.toString())
|
||||
map.put("strctureArea", size.toString())
|
||||
map.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB")
|
||||
|
||||
if (!address.contains("%")) {
|
||||
// 发送HTTP请求
|
||||
long start
|
||||
long end
|
||||
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(communityUrl, map, "UTF-8")
|
||||
//
|
||||
//
|
||||
// end = System.currentTimeMillis();
|
||||
// items['D'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(valueUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['E'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(caseUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['F'] = String.valueOf(end - start);
|
||||
//
|
||||
// start = System.currentTimeMillis();
|
||||
// httpRequest.getJsonContent(poiUrl, map, "UTF-8")
|
||||
// end = System.currentTimeMillis();
|
||||
// items['G'] = String.valueOf(end - start);
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
def content = httpRequest.getJsonContent(comprehensiveUrl, map, "UTF-8")
|
||||
end = System.currentTimeMillis();
|
||||
items['H'] = String.valueOf(end - start);
|
||||
println content.get("msg")
|
||||
if(content.get("msg").contains("该地址不支持估价!")) {
|
||||
items['I'] = "该地址不支持估价!"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
})
|
||||
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
package com.fangjia.script.majintao.test
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
count = count + 1
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
sleep(50)
|
||||
|
||||
def address = items['A']
|
||||
def map = SignatureUtils.getAliAddressStrc(address)
|
||||
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
|
||||
if ("prov" in map) {
|
||||
def province = map.get("prov")
|
||||
items['D'] = province
|
||||
}
|
||||
if ("city" in map) {
|
||||
def city = map.get("city")
|
||||
items['H'] = city
|
||||
}
|
||||
if ("district" in map) {
|
||||
def district = map.get("district")
|
||||
items['L'] = district
|
||||
}
|
||||
if ("town" in map) {
|
||||
def town = map.get("town")
|
||||
items['P'] = town
|
||||
}
|
||||
if ("community" in map) {
|
||||
def community = map.get("community")
|
||||
items['T'] = community
|
||||
|
||||
}
|
||||
// if ("road" in map) {
|
||||
// def road = map.get("road")
|
||||
// items['G'] = road
|
||||
//
|
||||
// }
|
||||
// if ("poi" in map) {
|
||||
//
|
||||
// def poi = map.get("poi")
|
||||
// items['H'] = poi
|
||||
//
|
||||
// }
|
||||
// if ("roomno" in map) {
|
||||
// def roomno = map.get("roomno")
|
||||
// items['I'] = roomno
|
||||
//
|
||||
// }
|
||||
// if ("devzone" in map) {
|
||||
// def devzone = map.get("devzone")
|
||||
// items['J'] = devzone
|
||||
//
|
||||
// }
|
||||
// if ("roadno" in map) {
|
||||
// def roadno = map.get("roadno")
|
||||
// items['K'] = roadno
|
||||
// }
|
||||
|
||||
|
||||
})
|
||||
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
package com.fangjia.script.majintao.test
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
count = count + 1
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
sleep(50)
|
||||
|
||||
def address = items['A']
|
||||
def pro = items['D']
|
||||
if (pro == null) {
|
||||
|
||||
def map = SignatureUtils.getAliAddressStrc(address)
|
||||
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
|
||||
if ("prov" in map) {
|
||||
def province = map.get("prov")
|
||||
items['D'] = province
|
||||
}
|
||||
if ("city" in map) {
|
||||
def city = map.get("city")
|
||||
items['H'] = city
|
||||
}
|
||||
if ("district" in map) {
|
||||
def district = map.get("district")
|
||||
items['L'] = district
|
||||
}
|
||||
if ("town" in map) {
|
||||
def town = map.get("town")
|
||||
items['P'] = town
|
||||
}
|
||||
if ("community" in map) {
|
||||
def community = map.get("community")
|
||||
items['T'] = community
|
||||
|
||||
}
|
||||
// if ("road" in map) {
|
||||
// def road = map.get("road")
|
||||
// items['G'] = road
|
||||
//
|
||||
// }
|
||||
// if ("poi" in map) {
|
||||
//
|
||||
// def poi = map.get("poi")
|
||||
// items['H'] = poi
|
||||
//
|
||||
// }
|
||||
// if ("roomno" in map) {
|
||||
// def roomno = map.get("roomno")
|
||||
// items['I'] = roomno
|
||||
//
|
||||
// }
|
||||
// if ("devzone" in map) {
|
||||
// def devzone = map.get("devzone")
|
||||
// items['J'] = devzone
|
||||
//
|
||||
// }
|
||||
// if ("roadno" in map) {
|
||||
// def roadno = map.get("roadno")
|
||||
// items['K'] = roadno
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
})
|
||||
|
||||
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.fangjia.script.majintao.test
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\ÁÙʱÊý¾Ý\\2.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
|
||||
String url = "http://192.168.0.19:31080/FangJiaAddressService/v3/structure";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
String address = items['A']
|
||||
|
||||
def map = [:]
|
||||
map.put("address", address)
|
||||
map.put("complete", "true")
|
||||
map.put("addressAssessment", "true")
|
||||
|
||||
def content = httpRequest.getJsonContent(url, map, "UTF-8")
|
||||
def result = content.get("result")
|
||||
if ("province" in result) {
|
||||
items['B'] = result.get("province")
|
||||
}
|
||||
if ("provinceCode" in result) {
|
||||
items['C'] = result.get("provinceCode")
|
||||
}
|
||||
if ("city" in result){
|
||||
items['D'] = result.get("city")
|
||||
}
|
||||
if("cityCode" in result){
|
||||
items['E'] = result.get("cityCode")
|
||||
}
|
||||
if ("district" in result){
|
||||
|
||||
items['F'] = result.get("district")
|
||||
}
|
||||
if ("districtCode" in result){
|
||||
|
||||
items['G'] = result.get("districtCode")
|
||||
}
|
||||
if ("township" in result){
|
||||
|
||||
items['H'] = result.get("township")
|
||||
}
|
||||
if ("townshipCode" in result){
|
||||
|
||||
items['I'] = result.get("townshipCode")
|
||||
print(result.get("townshipCode"))
|
||||
}
|
||||
if ("address" in result){
|
||||
|
||||
items['J'] = result.get("address")
|
||||
}
|
||||
if ("formatAddress" in result){
|
||||
items['K'] = result.get("formatAddress")
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package com.fangjia.script.majintao.test
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
|
||||
count += 1
|
||||
if (count % 10 == 0) {
|
||||
println("已处理:" + count)
|
||||
}
|
||||
|
||||
|
||||
String address = items['A']
|
||||
String province = items['B']
|
||||
if (province == null) {
|
||||
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
|
||||
def map = [:]
|
||||
map.put("address", address)
|
||||
map.put("complete", "true")
|
||||
map.put("addressAssessment", "true")
|
||||
map.put("type", "1")
|
||||
|
||||
def content = httpRequest.getJsonContent(url, map, "UTF-8")
|
||||
def result = content.get("result")
|
||||
if (result != null) {
|
||||
if ("province" in result) {
|
||||
items['B'] = result.get("province")
|
||||
}
|
||||
if ("provinceCode" in result) {
|
||||
items['C'] = result.get("provinceCode")
|
||||
}
|
||||
if ("city" in result) {
|
||||
items['D'] = result.get("city")
|
||||
}
|
||||
if ("cityCode" in result) {
|
||||
items['E'] = result.get("cityCode")
|
||||
}
|
||||
if ("district" in result) {
|
||||
|
||||
items['F'] = result.get("district")
|
||||
}
|
||||
if ("districtCode" in result) {
|
||||
|
||||
items['G'] = result.get("districtCode")
|
||||
}
|
||||
if ("town" in result) {
|
||||
|
||||
items['H'] = result.get("town")
|
||||
}
|
||||
if ("townCode" in result) {
|
||||
|
||||
items['I'] = result.get("townCode")
|
||||
}
|
||||
|
||||
if ("committee" in result) {
|
||||
|
||||
items['J'] = result.get("committee")
|
||||
}
|
||||
if ("committeeCode" in result) {
|
||||
|
||||
items['K'] = result.get("committeeCode")
|
||||
}
|
||||
if ("typeCode" in result) {
|
||||
|
||||
items['L'] = result.get("typeCode")
|
||||
}
|
||||
if ("typeName") {
|
||||
items['M'] = result.get("typeName")
|
||||
}
|
||||
if ("verifyFlag") {
|
||||
items['N'] = result.get("verifyFlag")
|
||||
}
|
||||
if ("addrVerify") {
|
||||
items['O'] = result.get("addrVerify")
|
||||
}
|
||||
if ("countyLevelCity") {
|
||||
items['P'] = result.get("countyLevelCity")
|
||||
}
|
||||
if ("countyGov") {
|
||||
items['Q'] = result.get("countyGov")
|
||||
}
|
||||
if ("communitySource") {
|
||||
items['R'] = result.get("communitySource")
|
||||
}
|
||||
|
||||
if ("administrativeTowns" in result) {
|
||||
items['T'] = result.get("administrativeTowns")
|
||||
}
|
||||
if ("devDistrict" in result) {
|
||||
items['V'] = result.get("devDistrict")
|
||||
}
|
||||
if ("devDistrictName" in result) {
|
||||
items['W'] = result.get("devDistrictName")
|
||||
}
|
||||
if ("splitCommittee" in result) {
|
||||
items['X'] = result.get("splitCommittee")
|
||||
}
|
||||
if ("splitStreet" in result) {
|
||||
items['Y'] = result.get("splitStreet")
|
||||
}
|
||||
if ("poiName" in result) {
|
||||
items['Z'] = result.get("poiName")
|
||||
}
|
||||
if ("splitStreetNum" in result) {
|
||||
items['AA'] = result.get("splitStreetNum")
|
||||
}
|
||||
if ("dev" in result) {
|
||||
items['AB'] = result.get("dev")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
package com.fangjia.script.majintao.test
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
|
||||
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\对公.xlsx";
|
||||
ExcelUtils.removeEmptyRow(excelFilePath)
|
||||
def count = 0
|
||||
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
|
||||
count+=1
|
||||
if (count%10==0){
|
||||
println("已处理:"+count)
|
||||
}
|
||||
|
||||
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
|
||||
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
map = new HashMap<String, String>()
|
||||
|
||||
String address = items['A']
|
||||
// print(address)
|
||||
|
||||
// province;
|
||||
// city;
|
||||
// district;
|
||||
// town;
|
||||
// committee;
|
||||
// type;
|
||||
// provinceCode;
|
||||
// cityCode;
|
||||
// districtCode;
|
||||
// townCode;
|
||||
// committeeCode;
|
||||
// typeCode;
|
||||
// typeName;
|
||||
// verifyFlag;//涉农标识
|
||||
// addrVerify;//地址是否有效信息
|
||||
// countyLevelCity;//是否为县级市
|
||||
// administrativeTowns;//是否市辖建制镇
|
||||
// countyGov;//是否为县政府
|
||||
// devDistrict;//是否为开发区
|
||||
// devDistrictName;//开发区
|
||||
// splitCommittee;
|
||||
// splitStreet;
|
||||
// poiName;
|
||||
// splitStreetNum;
|
||||
// dev;
|
||||
// communitySource;//居委来源
|
||||
|
||||
def map = [:]
|
||||
map.put("address", address)
|
||||
map.put("complete", "true")
|
||||
map.put("addressAssessment", "true")
|
||||
map.put("type", "2")
|
||||
|
||||
def content = httpRequest.getJsonContent(url, map, "UTF-8")
|
||||
def result = content.get("result")
|
||||
if ("province" in result) {
|
||||
items['B'] = result.get("province")
|
||||
}
|
||||
if ("provinceCode" in result) {
|
||||
items['C'] = result.get("provinceCode")
|
||||
}
|
||||
if ("city" in result){
|
||||
items['D'] = result.get("city")
|
||||
}
|
||||
if("cityCode" in result){
|
||||
items['E'] = result.get("cityCode")
|
||||
}
|
||||
if ("district" in result){
|
||||
|
||||
items['F'] = result.get("district")
|
||||
}
|
||||
if ("districtCode" in result){
|
||||
|
||||
items['G'] = result.get("districtCode")
|
||||
}
|
||||
if ("town" in result){
|
||||
|
||||
items['H'] = result.get("town")
|
||||
}
|
||||
if ("townCode" in result){
|
||||
|
||||
items['I'] = result.get("townCode")
|
||||
}
|
||||
|
||||
if ("committee" in result){
|
||||
|
||||
items['J'] = result.get("committee")
|
||||
}
|
||||
if ("committeeCode" in result){
|
||||
|
||||
items['K'] = result.get("committeeCode")
|
||||
}
|
||||
if ("typeCode" in result){
|
||||
|
||||
items['L'] = result.get("typeCode")
|
||||
}
|
||||
if ("typeName"){
|
||||
items['M'] = result.get("typeName")
|
||||
}
|
||||
if ("verifyFlag"){
|
||||
items['N'] = result.get("verifyFlag")
|
||||
}
|
||||
if ("addrVerify"){
|
||||
items['O'] = result.get("addrVerify")
|
||||
}
|
||||
if ("countyLevelCity"){
|
||||
items['P'] = result.get("countyLevelCity")
|
||||
}
|
||||
if ("countyGov"){
|
||||
items['Q'] = result.get("countyGov")
|
||||
}
|
||||
if ("communitySource"){
|
||||
items['R'] = result.get("communitySource")
|
||||
}
|
||||
|
||||
if ("administrativeTowns" in result){
|
||||
items['T'] = result.get("administrativeTowns")
|
||||
}
|
||||
if ("devDistrict" in result){
|
||||
items['V'] = result.get("devDistrict")
|
||||
}
|
||||
if ("devDistrictName" in result){
|
||||
items['W'] = result.get("devDistrictName")
|
||||
}
|
||||
if ("splitCommittee" in result){
|
||||
items['X'] = result.get("splitCommittee")
|
||||
}
|
||||
if ("splitStreet" in result){
|
||||
items['Y'] = result.get("splitStreet")
|
||||
}
|
||||
if ("poiName" in result){
|
||||
items['Z'] = result.get("poiName")
|
||||
}
|
||||
if ("splitStreetNum" in result){
|
||||
items['AA'] = result.get("splitStreetNum")
|
||||
}
|
||||
if ("dev" in result){
|
||||
items['AB'] = result.get("dev")
|
||||
}
|
||||
|
||||
|
||||
|
||||
})
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user