This commit is contained in:
2024-03-01 10:26:00 +08:00
185 changed files with 20889 additions and 5967 deletions
@@ -2,36 +2,35 @@ package com.fangjia.script
public enum Watcher {
//ÀëְͬÊÂ
LiuPeiPei('13621626685', 'duxiaoyu@fangjia.com'),
YanZhou('13621626685','duxiaoyu@fangjia.com'),
HuangXiaoHu('13621626685', 'duxiaoyu@fangjia.com'),
SongYingXia('13621626685', 'duxiaoyu@fangjia.com'),
YuKai('13621626685', 'duxiaoyu@fangjia.com'),
RuanYaDong('13621626685', 'duxiaoyu@fangjia.com'),
Reda('13621626685', 'duxiaoyu@fangjia.com'),
Tomzhao('13621626685','duxiaoyu@fangjia.com'),
YuHai('13621626685','duxiaoyu@fangjia.com'),
Shibb('13621626685','duxiaoyu@fangjia.com'),
HeLvYuan('13621626685','duxiaoyu@fangjia.com'),
WangYuHui('13621626685','duxiaoyu@fangjia.com'),
LinXiaoShuai('15801810393','linxiaoshuai@fangjia.com'),
FangRui('17717680589','fangrui@fangjia.com'),
//DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'),
LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'),
YanZhou('13621626685','wangwenyan@fangjia.com'),
HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'),
SongYingXia('13621626685', 'wangwenyan@fangjia.com'),
YuKai('13621626685', 'wangwenyan@fangjia.com'),
RuanYaDong('13621626685', 'wangwenyan@fangjia.com'),
Reda('13621626685', 'wangwenyan@fangjia.com'),
Tomzhao('13621626685','wangwenyan@fangjia.com'),
YuHai('13621626685','wangwenyan@fangjia.com'),
Shibb('13621626685','wangwenyan@fangjia.com'),
HeLvYuan('13621626685','wangwenyan@fangjia.com'),
WangYuHui('13621626685','wangwenyan@fangjia.com'),
LinXiaoShuai('15801810393','wangwenyan@fangjia.com'),
FangRui('17717680589','wangwenyan@fangjia.com'),
DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
//ÔÚְͬÊÂ
DuXiaoYu('13916076378', 'wangwenyan@fangjia.com'),
LiXue('13311649646','lixue@fangjia.com'),
LiXue('13311649646','wangwenyan@fangjia.com'),
WangWenYan('13916076378','wangwenyan@fangjia.com'),
GaoJuan('15021384995','gaojuan@fangjia.com'),
Hubingliang('15179856208','hubingliang@126.com'),
ZhouTongYun('13585834921','zhoutongyun@fangjia.com'),
YinJiHuan('18217642009','yinjihuan@fangjia.com'),
Xuhe('13862416422','xuhe@fangjia.com'),
ZongZhiMin('18260046486','zongzhimin@fangjia.com'),
RaoYanHui('18516539752','raoyanhui@fangjia.com'),
BaYi('15300424133','bayi@fangjia.com'),
WeiLei('18101751850','weilei@fangjia.com'),
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
Hubingliang('15179856208','wangwenyan@fangjia.com'),
ZhouTongYun('13585834921','wangwenyan@fangjia.com'),
YinJiHuan('18217642009','wangwenyan@fangjia.com'),
Xuhe('13862416422','wangwenyan@fangjia.com'),
ZongZhiMin('18260046486','wangwenyan@fangjia.com'),
RaoYanHui('18516539752','wangwenyan@fangjia.com'),
BaYi('15300424133','wangwenyan@fangjia.com'),
WeiLei('18101751850','wangwenyan@fangjia.com'),
JiangYaHui('13684913780','wangwenyan@fangjia.com'),
ShenJian('13764369683', 'shenjian@fangjia.com')
private final String mobile;
private final String email;
@@ -124,13 +124,13 @@ public class ManualOperationDataStat extends BaseBatch{
println(content)
sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
}
}
@@ -1,5 +1,7 @@
package com.fangjia.script.ana
import com.fangjia.script.util.ExcelUtils
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -40,9 +42,46 @@ public class Test {
// //test.run("district", "上海");
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
// println new Date(new ObjectId('5469bb4cda29ee73010fa577').getTime())
test.addTestData();
// test.addTestData();
test.findRepeatGuidingPrice()
}
def findRepeatGuidingPrice(){
def q = new BasicDBObject();
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-03-01", "yyyy-MM-dd")))
def dbcursor = GUIDING_PRICE_COLL.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.name
def rets = map.get(key)
if(!rets){
rets = []
map.put(key, rets)
}
rets << d
}
def datas = []
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
map.each{key,list->
if(list.size()>1){
def userNames = []
for(def ret : list){
userNames << ret.guiding_person
}
if(userNames.contains("付贵生")){
for(def r : list){
if(r.guiding_person!="付贵生"){
GUIDING_PRICE_COLL.remove(new BasicDBObject("_id", r._id))
}
}
}
}
}
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
}
public void addTestData(){
@@ -28,7 +28,8 @@ import org.bson.types.ObjectId
class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017)
@@ -215,7 +216,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("cat", "district")
query.put("visible", 0)
query.put("status", 0)
DBObject fields = new BasicDBObject()
fields.put("city", 1)
@@ -227,11 +228,9 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
int tryTimes = 5
List<BasicDBObject> districts = new ArrayList<BasicDBObject>()
for(int i=0; i < tryTimes; i++) {
DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
def cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
int totalCount = cur.count()
Iterator<DBObject> iter = cur.iterator()
while(iter.hasNext()) {
BasicDBObject dbo = (BasicDBObject)iter.next()
for(def dbo : cur) {
districts.add(dbo)
}
if(districts.size() == totalCount) {
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -58,10 +58,16 @@ for(def d : datas){
}catch(Exception e){
println "room error"
}
map.hall = arrs[1].toString().toInteger()
map.toilet = arrs[3].toString().toInteger()
if(arrs[1].toString()){
map.hall = arrs[1].toString().toInteger()
}
if(arrs.size()==4 && arrs[3].toString()){
map.toilet = arrs[3].toString().toInteger()
}
}else{
map.room = roomStr.toString().toDouble().toInteger()
if(roomStr.toString()){
map.room = roomStr.toString().toDouble().toInteger()
}
}
}
@@ -74,7 +80,7 @@ for(def d : datas){
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
}
@@ -0,0 +1,77 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 添加人工指导价,并重算价格
* 小付的人工指导价不能覆盖
*/
class BatchAddPersonGuiPrice {
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
static main(args) {
def task = new BatchAddPersonGuiPrice()
task.run()
}
def run(){
def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx"
def datas = ExcelUtils.read(path, 0, 1)
for(def d : datas){
def row = findGuidingPrice(d['B'], d['C'], d['D'])
if(!row){
def obj = CommonDatautils.findDistrictById(d['A'])
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2)
//发重算均价命令
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date())
}
}
}
def findGuidingPrice(city, region, name) {
def q = new BasicDBObject();
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("guiding_person" , "付贵生")
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd")))
def dbo = GUIDING_PRICE_COLL.findOne(q)
return dbo
}
def save(city, region, name, price, estate_type){
DBObject obj = new BasicDBObject();
obj.put("city", city);
obj.put("region", region);
obj.put("name", name);
obj.put("guiding_date", new Date()-7);
obj.put("guiding_value", price.toString().toDouble().toInteger());
obj.put("category", "district");
obj.put("guiding_type", 0);
obj.put("c_date", new Date());
obj.put("m_date", new Date());
obj.put("estate_type" , estate_type);
obj.put("house_type" , 0)
obj.put("guiding_person" , "陈杰")
GUIDING_PRICE_COLL.save(obj);
}
def addPrice(city, region, name, estateType2,startDate, endDate){
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
}
}
}
@@ -90,11 +90,16 @@ class BigdataCommunityBatch extends BaseBatch{
if(today > 10){
delPgData()
saveToPgSql(rows)
//更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
// saveToPgSql2(datas)
}
def addDeleteDist(dists, ccbMaps, rets){
@@ -311,7 +316,7 @@ class BigdataCommunityBatch extends BaseBatch{
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def index = 0
sqlInstance.withBatch(10000, sql){ ps->
sqlInstance.withBatch(50000, sql){ ps->
for(def r : newRows) {
index++
try{
@@ -339,4 +344,12 @@ class BigdataCommunityBatch extends BaseBatch{
return rows;
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
sqlInstance.executeUpdate(sql)
}
}
}
@@ -39,7 +39,7 @@ class BigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
@@ -60,20 +60,14 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
def citys = CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -84,7 +78,7 @@ class BigdataDealBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def dealIds = findHistoryDealIds(city)
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
@@ -115,11 +109,29 @@ class BigdataDealBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
// delPgData()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
// saveToPgSql2(datas)
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("\\|@\\|")
rows << row
}
return rows;
}
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
def result = Collections.synchronizedList(new ArrayList());
dealIds = Collections.synchronizedList(dealIds);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
@@ -147,12 +159,14 @@ class BigdataDealBatch extends BaseBatch{
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}
}else{
def ret = converData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
}
@@ -167,6 +181,7 @@ class BigdataDealBatch extends BaseBatch{
}
println "查询完成......."
latch.await(5, TimeUnit.MINUTES);
return result
}
@@ -217,7 +232,7 @@ class BigdataDealBatch extends BaseBatch{
def price = ccbPriceMap.get(obj._id.toString())
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){
if(mod<=20){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
@@ -515,6 +530,7 @@ class BigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -635,7 +651,7 @@ class BigdataDealBatch extends BaseBatch{
}
def findHistoryDealIds(def city){
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}' "
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'"
def ids = []
try{
pgsqlInstance.eachRow(sql,{
@@ -723,6 +739,14 @@ class BigdataDealBatch extends BaseBatch{
println city +" "+trades.size()+" "+new Date().toString()
}
def delPgData(){
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
pgsqlInstance.execute("delete from base.t0831_bigdata_deal_gj_h where p9_start_date='${month}'")
println "base.t0831_bigdata_deal_gj_h 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
@@ -753,4 +777,43 @@ class BigdataDealBatch extends BaseBatch{
println "end: "+new Date()
}
def saveToPgSql2(rows){
def newRows = []
for(def r : rows) {
def newR = []
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
// r[i] = null
newR << null
}else{
newR << r[i]
}
}
newR << null
newR << "2023-09-28"
newR << null
newR << '2999-12-31'
newR << null
newR << null
newR << null
newRows << newR
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : newRows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
}
}
println "end: "+new Date()
}
}
@@ -85,6 +85,7 @@ class BigdataForsaleBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def startDate = lastMonth+"18"
def endDate = curMonth+"19"
println "房源时间:"+startDate+" "+endDate
@@ -196,6 +197,7 @@ class BigdataForsaleBatch extends BaseBatch{
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
@@ -311,7 +313,7 @@ class BigdataForsaleBatch extends BaseBatch{
data << m.floor==null?"":m.floor
data << m.height==null?"":m.height
data << m.estate_type==null?"":m.estate_type
data << m.house_structure_code==null?"":m.house_structure_code
data << m.room_structure_code==null?"":m.room_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
data << ""
@@ -670,6 +672,7 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
@@ -731,7 +734,7 @@ class BigdataForsaleBatch extends BaseBatch{
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
if(!alias){
alias = []
@@ -882,6 +885,7 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
@@ -0,0 +1,192 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
/***
* 同步上个月城市,区域,小区价格到base库中
*/
class BigdataSynchPrice extends BaseBatch{
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) {
runBatch(new BigdataSynchPrice(), args)
}
@Override
void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
synchCityPrice(citys)
synchDistrictPrice(citys)
synchCommPirce(citys)
}
def synchCityPrice(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_city_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步城市均价 start: "+new Date()
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(500, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def synchDistrictPrice(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'"; //
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gxda_admndiv_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步区域均价 start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def synchCommPirce(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_comm_basic_prc
data << r.gujia_bds_clmtd
data << r.gujiabds_fwd_samp_rto
data << r.gujia_bds_mns_grd
data << r.gujia_bds_cfdrg
data << r.gujiabdsarndcmprccrlt
data << r.gujia_bds_cfdc_dgr
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步小区价格 start: "+new Date()
def sql = "insert into base.t0831_community_base_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(50000, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
}
@@ -30,6 +30,9 @@ class CcbCommonUtils {
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
@@ -37,7 +40,7 @@ class CcbCommonUtils {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42","xflapp.com":"43"]
public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"]
@@ -225,7 +228,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -432,7 +435,7 @@ class CcbCommonUtils {
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","金地玺湾")
// q.put("name","汤堡小区")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -443,7 +446,7 @@ class CcbCommonUtils {
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
@@ -467,7 +470,7 @@ class CcbCommonUtils {
public static def findAllDistrictByCityAndRegion(def city, def region, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", region)
@@ -482,7 +485,7 @@ class CcbCommonUtils {
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "原始:"+dbsour.count()
def rets = []
for(def d : dbsour){
@@ -679,4 +682,24 @@ class CcbCommonUtils {
return ids;
}
static def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
}
@@ -29,7 +29,7 @@ class DealPriceDataProcess {
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
task.deleteThirdPartyData()//删除第三方
// task.delDealByPe()
// task.delDealByPe()//删除成交
// task.delDealData()
// task.exportThirdParty()
@@ -37,49 +37,49 @@ class DealPriceDataProcess {
}
def delDealByPe(){
def yearMonth = 202201
def thirdMonth = 202112
def yearMonth = 202401
// def rets = findAllDealByPe(yearMonth, 70)
// for(def r : rets){
// deleteDealPrice(r)
def rets = findAllDealByPe(yearMonth, 18)
for(def r : rets){
deleteDealPrice(r)
}
// def list = findAllThirdByPe(yearMonth, 70)
// for(def r : list){
// deletThirdPrice(r, thirdMonth)
// }
// def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
// for(def d : datas){
//// if(d['G']){
//// def rets = findDealByPe(d['A'], yearMonth, d['G'])
//// for(def r : rets){
//// deleteDealPrice(r)
//// }
//// }
//
//// if(d['H']){
//// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
//// for(def m : rets){
//// def precent = d['H']/100.0
//// def newPrice = null
//// if(m.unit_price > m.deal_price){
//// newPrice = m.deal_price*precent + m.deal_price
//// }else if(m.unit_price < m.deal_price){
//// newPrice = m.deal_price - (m.deal_price*precent)
//// }
//// if(newPrice){
//// updateDealPrice(m, newPrice.toInteger())
//// }
//// }
//// }
//
//// if(d['M']){
//// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
//// for(def r : rets){
//// deletThirdPrice(r, thirdMonth)
//// }
//// }
// }
def list = findAllThirdByPe(yearMonth, 70)
for(def r : list){
deletThirdPrice(r, thirdMonth)
}
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
for(def d : datas){
// if(d['G']){
// def rets = findDealByPe(d['A'], yearMonth, d['G'])
// for(def r : rets){
// deleteDealPrice(r)
// }
// }
// if(d['H']){
// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
// for(def m : rets){
// def precent = d['H']/100.0
// def newPrice = null
// if(m.unit_price > m.deal_price){
// newPrice = m.deal_price*precent + m.deal_price
// }else if(m.unit_price < m.deal_price){
// newPrice = m.deal_price - (m.deal_price*precent)
// }
// if(newPrice){
// updateDealPrice(m, newPrice.toInteger())
// }
// }
// }
// if(d['M']){
// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
// for(def r : rets){
// deletThirdPrice(r, thirdMonth)
// }
// }
}
}
@@ -151,9 +151,10 @@ class DealPriceDataProcess {
}
def deleteDealPrice(r){
def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
"and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
sqlInstance236.executeUpdate(sql)
// def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
// "and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
def sql = "delete from base.t0831_bigdata_deal_gj_h where gujia_comm_id='${r.commid}' and txn_dt='${r.dealdate}' and guiabconstructionarea=${r.buildarea} and gujia_bds_trdtckt_prc=${r.deal_price}"
pgsqlInstance.executeUpdate(sql)
}
def deletThirdPrice(r, thirdMonth){
@@ -177,19 +178,14 @@ class DealPriceDataProcess {
}
def findAllDealByPe(yearMonth, peVal){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='成交' and pe>${peVal}";
def sql = "select gujia_comm_id as commid,gujiabds_valt_unitprc as unit_price,gujiabds_to_mkprc_mny as target_avg_price,gujia_area as buildarea,to_char(mdl_dt,'yyyy-mm-dd') as dealdate from t_app_p9pha_plda.avm_evaluation_item_gj_h where urbn_nm in ('昆明','南京','苏州','重庆') and gujiabds_clc_yrmo_val=${yearMonth} and gujia_data_tp='成交' and gujia_mns_rto>${peVal} ";
def rets = []
sqlInstance236.eachRow(sql,{r->
pgsqlInstance.eachRow(sql,{r->
def map = [:]
map.cityname = r.cityname
map.cityid=r.cityid
map.distid = r.distid
map.commid = r.commid
map.commname = r.commname
map.deal_price = r.target_avg_price
map.unit_price = r.unit_price
map.buildarea = r.buildarea
map.ppe = r.pe
map.dealdate = r.dealdate
rets << map
})
@@ -392,12 +388,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202306
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202307)
def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
def total = 0
println city +" "+rets.size()
if(rets){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -84,8 +84,7 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//
//更新静态数据表
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -244,14 +243,7 @@ class GuiDingPriceBatch extends BaseBatch{
println "end: "+new Date()
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
pgsqlInstance.executeUpdate(sql)
}
}
@@ -2,6 +2,7 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.RandomUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
@@ -1287,10 +1288,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
@@ -1299,37 +1300,42 @@ def fileName = "D:\\
// d['H'] = dist._id.toString()
// }
// }
//// if(!d['F']){
//// def dist = CommonDatautils.findDistrictById(d['A'])
//////
//// if(dist){
////// d['H'] = dist.region
////// d['G'] = dist.name
//// d['F'] = dist.status
//// }
//// }
//
// }
//},0)
if(!d['I']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
d['F'] = dist.region
// d['H'] = dist.name
d['G'] = dist.status
// if(d['A']=="天门" || d['A']=="迪庆"){
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
// d['K'] = val.toInteger()
// }
}
}
}
},0)
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
// if(count && count==1){
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
// d['E'] = dist.city
//// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
//// if(count && count==1){
//// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
//// d['E'] = dist.city
//// }
////
// if(d['F']){
// def dist = CommonDatautils.findDistrictById(d['F'])
// if(dist){
// d['G'] = dist.status
// d['H'] = dist.name_cat
// d['I'] = dist.estate_type2
// }
// }
//
//// def dist = CommonDatautils.findDistrictById(d['A'])
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['J'] = dist.status
//// }
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
//// d['F'] = newRegion
////// def newRegion = RegionData.formatRegion(d['A'], d['B'])
////// d['F'] = newRegion
//},0)
//ExcelUtils.modify(fileName, {d->
@@ -1373,30 +1379,30 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
return count
}
citys = CcbCommonUtils.citys_all
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
def rows = []
def headers = []
headers << "城市"
headers.addAll(dates)
rows << headers
for(def c : citys){
def row = []
row << c
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
// for(def d : dates){
def startDate = "2022-07-01"
def count = findPgDealPrice(c, pgsqlInstance, startDate)
row << count*100/disCount
// def map = findSampleRate(pgsqlInstance, c, d)
// row << map.third
// }
rows << row
}
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
//citys = CcbCommonUtils.citys_all
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
//def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
//def rows = []
//def headers = []
//headers << "城市"
//headers.addAll(dates)
//rows << headers
//for(def c : citys){
// def row = []
// row << c
// def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
//// for(def d : dates){
// def startDate = "2022-07-01"
// def count = findPgDealPrice(c, pgsqlInstance, startDate)
// row << count*100/disCount
//
//// def map = findSampleRate(pgsqlInstance, c, d)
//// row << map.third
//// }
// rows << row
//}
//ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
@@ -1414,6 +1420,31 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\房源\\仙桃.xlsx", {d->
// def dist = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
// def flag = null;
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'],d['B'],d['C'],"district")
// flag = "映射别名匹配"
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(d['A'],d['B'],d['C'], "district")
// if(rets && rets.size()==1){
// flag = "别名匹配"
// dist = rets[0]
// }
// }
// }
// if(dist){
// d['D'] = dist.status
// d['E'] = dist.estate_type2
// d['F'] = flag
// d['G'] = dist.name
// }
//
//})
@@ -54,7 +54,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
// ps.addBatch(id)
// }
// }
//
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年10月总量","23年11月总量","23年12月总量","占比","贝壳","链家", "我爱我家","幸福里APP","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-04-01"
def date = "2023-05-01"
def endDate = "2023-06-01"
def newDate = "2023-07-01"
def startDate = "2023-10-01"
def date = "2023-11-01"
def endDate = "2023-12-01"
def newDate = "2024-01-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -499,8 +499,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2022-03-01"
def endDate = "2022-04-01"
def startDate = "2023-08-01"
def endDate = "2023-10-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -37,44 +37,47 @@ class ExportAllPgData {
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
////
//// //第三方网站权重表
//////
////// //第三方网站权重表
// exportWebsiteRate();
//
////
// exportGuidedPrice()//随人工改变//
// exportCommunityGuidedPrice()//随政策//
// //处理小区情况
//// //处理小区情况
// exportCommunityStaging()//没cityid
// exportBigdataCommunity()//
//
////
// exportCommunityBuildType()//
//
exportThirdPartyPrice2(new_dists)//
exportBigdataForsale(new_dists) //现在按updatetime导出
////
// exportThirdPartyPrice2(new_dists)//
// exportBigdataForsale(new_dists) //现在按updatetime导出
// exportBigdataDeal(new_dists) //2022-07-01
//计算结果相关
//
//
// //计算结果相关
// exportAvmEvaluationGJNB()
// exportAVMEvaluationItemGJNB()
//
////
// exportCityPrice()
// exportRegionPrice()
exportCommunityBasePrice()
exportCommunityPdescribeGj()
exportCommCaseDetailGj()
// exportCommunityBasePrice()
// exportCommunityBasePrice1(curMonth)
// exportCommunityPdescribeGj()
// exportCommCaseDetailGj()
//
// exportInOk()
exportInOk()
////
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
}
def exportEvaluationRateData() {
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h";
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from base.t0831_evaluate_rate_gj_h";
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
@@ -102,7 +105,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'";
def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
@@ -133,7 +136,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'";
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from base.t0831_bigdata_district_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_admndiv_id
@@ -164,7 +167,7 @@ class ExportAllPgData {
}
def exportWebsiteRate(){
def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h";
def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_website_config_gj_h";
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
@@ -194,7 +197,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'";
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
@@ -227,7 +230,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}'";
def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from base.t0831_community_guide_price_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
@@ -307,7 +310,7 @@ class ExportAllPgData {
def exportCommunityStaging() {
def allRows = []
def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h";
def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_community_staging_gj_h";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_instm_id
@@ -330,7 +333,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'";
def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from base.t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
@@ -363,12 +366,12 @@ class ExportAllPgData {
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
@@ -424,17 +427,17 @@ class ExportAllPgData {
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";
allRows.addAll(dealData(sql))
}
}
@@ -535,7 +538,7 @@ class ExportAllPgData {
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}";
"from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
@@ -560,7 +563,7 @@ class ExportAllPgData {
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
"from base.t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
def rets2 = getHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true);
@@ -17,7 +17,7 @@ class ExportPgsqlBaseEvaluation {
def database = "base"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\"
static main(args) {
@@ -41,11 +41,10 @@ class ExportPgsqlEvaluationResult {
exportRegionPrice(curMonth)
exportCommunityBasePrice(curMonth)
// exportCommunityBasePrice1(curMonth)
exportCommunityPdescribeGj(curMonth)
exportCommCaseDetailGj(curMonth)
//
exportInOk()
//
SSHHelperTest jschCommand = new SSHHelperTest();
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
// cityRun(citys, startDate, endDate, cityCodeMap)
// regionRun(citys, startDate, endDate, cityCodeMap)
cityRun(citys, startDate, endDate, cityCodeMap)
regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
@@ -52,6 +52,8 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportGuidedPrice()//10月10号再同步数据
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
@@ -188,6 +190,46 @@ class ExportDistrictRongHe extends BaseBatch{
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMUNITY_FJW ÉÏ´«FtpÍê³É......"
}
def exportGuidedPrice() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_bds_id
data << r.comm_nm
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_drc_dt
data << r.gujia_bds_drc_prc
data << r.gujia_add_psn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "GUIDED_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "GUIDED_PRICE_GJ 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def allRows = []
for(def city : citys) {
@@ -19,10 +19,12 @@ class PocTest {
static main(args) {
def task = new PocTest();
task.run();
// task.run();
// task.export()
// task.distance()
}
def export(){
ExcelUtils.modify("D:\\Êý¾Ý\\ÆäËü\\¹ãÖÝ.xlsx", { row ->
def dist = CommonDatautils.findDistrictById(row["B"])
@@ -156,7 +158,6 @@ class PocTest {
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
// row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['R'] = priceObj.avg_price
@@ -1,8 +1,12 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import groovy.json.JsonSlurper
@@ -32,6 +36,7 @@ import org.elasticsearch.index.query.QueryBuilders
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
class CommonDatautils {
@@ -83,8 +88,17 @@ class CommonDatautils {
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
public static final DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
static def DIRECTIONS = ["¶«±±", "Î÷±±", "¶«ÄÏ", "Î÷ÄÏ", "Äϱ±", "¶«Î÷", "¶«", "ÄÏ", "Î÷", "±±"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
@@ -227,12 +241,11 @@ class CommonDatautils {
public static def findRegionDistrictCount(def cat, def city, def region){
DBObject q = new BasicDBObject();
q.put("visible", 0);
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
q.put("region", region);
DSCursor cursor = getDataStoreConnection().find(q);
return cursor.count();
return DbUtils.seaweed.count(q);
}
public static def findAllFiledDistrict(def cat, def city){
@@ -246,12 +259,13 @@ class CommonDatautils {
List<DBObject> dbs = cursor.getResults();
return dbs;
}
public static def findAllDistricts(def city, def cat, def fields){
public static def findAllDistrictsFromMg(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible", 0);
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","泰瑞家园")
DBObject keys = new BasicDBObject();
if(fields){
@@ -260,7 +274,7 @@ class CommonDatautils {
}
}
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -298,7 +312,11 @@ class CommonDatautils {
q.put("cat", cat);
q.put("city", city);
if(startDate){
q.put("m_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def datequery = new BasicDBObject('$gte', startDate)
if(endDate){
datequery.append('$lt',endDate)
}
q.put("m_date", datequery)
}
DBObject keys = new BasicDBObject();
@@ -784,10 +802,33 @@ class CommonDatautils {
}
public static def findAllDistricts(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city)
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllDistricMaptByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
@@ -797,7 +838,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def retMap = [:]
for(def d : dbsour){
retMap.put(d._id.toString(), d)
@@ -806,27 +847,27 @@ class CommonDatautils {
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","Íò´ï»ª¸®Ò»ÆÚ")
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
// public static def findAllDistrictByCity(def city, def cat, def fields){
//
// DBObject q = new BasicDBObject();
// q.put("status",0)
// q.put("cat", cat);
// q.put("city", city);
//// q.put("name","万达华府一期")
//
// DBObject keys = new BasicDBObject();
// if(fields){
// for(def field : fields){
// keys.put(field, 1);
// }
// }
// def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
// def rets = []
// for(def d : dbsour){
// rets << d
// }
// return rets;
// }
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
@@ -1731,14 +1772,14 @@ class CommonDatautils {
}
public static def queryDistrictPrice(city, region, name, weekend){
public static def queryFirstDistrictPrice(city, region, name){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
// queryCondition.put("weekend", weekend);
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", -1)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def dbo = null;
while(cur.hasNext()) {
@@ -2206,6 +2247,21 @@ class CommonDatautils {
}
public static def queryAllDistrictPrice(city, startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put('weekend', startDate);
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rMap = [:];
while(cur.hasNext()) {
DBObject dbo = cur.next()
rMap.put(city + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
}
return rMap;
}
public static def queryAllDistrictPrice(startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put('weekend', startDate);
@@ -2604,6 +2660,7 @@ class CommonDatautils {
q.put("cat", cat);
q.put('lat2', new BasicDBObject(QueryOperators.LTE, maxLat).append(QueryOperators.GTE, minLat));
q.put('lng2', new BasicDBObject(QueryOperators.LTE, maxLng).append(QueryOperators.GTE, minLng));
q.put("visible", 0)
def cur = getDataStoreConnection().find(q);
def ret = [];
for(def obj : cur){
@@ -3862,42 +3919,6 @@ class CommonDatautils {
return avg_price
}
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("lianjia_esf_price", db.beike_esf_price)
}
}
return map
}
/**
@@ -3941,5 +3962,381 @@ class CommonDatautils {
return files;
}
static def findLogs(userId, requestUri, startDate, endDate, code){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
if(startDate){
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
queryBuilder.must(date_range_cdate);
}
if(code){
queryBuilder.must(QueryBuilders.termQuery("code", code))
}
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
return rets.size()
}
static def findLogsBySql(userId, requestUri, startDate, endDate, code){
String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' "
if(startDate){
sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'"
}
if(code){
sql = sql + " and code=${code}"
}
def total = null
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
return total;
}
static def findNewAmapDetailBySql(city){
String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
def point = data.location
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
rets << ["data_id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location]
})
return rets;
}
static def findAmapDetailBySql(city){
String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
// if(data.name.contains('华山小区G栋')){
def point = data.location
// def tag = null
// if(data.get("business.keytag")){
// tag = data.get("business.keytag")
// }
def business_area = null
if(data.get("business.business_area")){
business_area = data.get("business.business_area")
}
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
rets << ["id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
// }
})
return rets;
}
//用sql有些字段取不出来
static def findDetailLogsBySql(userId, requestUri){
String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200"
sql = sql + " order by c_date desc"
def total = null
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
def ret = []
println data.toString()
ret << data.get("parameter_map.city")
// ret << data.get("parameter_map.district")
// ret << data.get("parameter_map.keyword")
// ret << data.get("parameter_map.name")
ret << data.get("parameter_map.address")
// ret << data.get("parameter_map.size")
ret << data.get("msg")
ret << data.get("code")
ret << data.get("c_date").toString()[0..10]
rets << ret
})
return rets;
}
static def findNewAmapDetail(city) {
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city))
def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder);
def results = []
if(rets) {
for (int k = 0; k < rets.size(); k++) {
def data = rets[k];
println data
}
}
return results;
}
static def findLogs2(userId, requestUri){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
queryBuilder.mustNot(QueryBuilders.termQuery("code", 200))
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def results = []
if(rets) {
for (int k = 0; k < rets.size(); k++) {
def data = rets[k];
println data
def ret = []
println data.toString()
ret << data.parameter_map.city
ret << data.parameter_map.district
ret << data.parameter_map.name
ret << data.parameter_map.address
ret << data.msg
ret << data.code
ret << data.c_date.toString()[0..10]
results << ret
}
}
return results
}
static def httpSaveData(def newUrl,def params){
def header = [:]
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
doPost(newUrl,header, params, true, "utf-8")
}
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setUseCaches(false);
conn.setConnectTimeout(5000);
conn.setReadTimeout(5000);
conn.setRequestProperty("User-Agent", UserAgents.get());
String param = null;
BufferedWriter mWriter = null;
InputStreamReader inStream = null;
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
conn.setRequestProperty(head, headers.get(head));
}
} else {
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
}
if (isJson) {
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
param = JacksonUtil.beanToJson(params)
}
conn.connect();
//println param.toString()
//输出流
if (StringUtils.isNotEmpty(param)) {
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
mWriter.write(param);
mWriter.flush();
mWriter.close();
}
if (conn.getHeaderField("Content-Encoding") != null
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
} else {
inStream = new InputStreamReader(conn.getInputStream(), charset);
}
char[] bf = new char[1024];
int len;
while ((len = inStream.read(bf)) != -1) {
buffer.append(bf, 0, len);
}
} catch (Exception e) {
e.printStackTrace()
}
return buffer.toString();
}
/**
* 获取小区周边一公里小区数量
* @param city
* @param lng
* @param lat
* @return
*/
static getDistrictArround(city, lng, lat) {
try {
def query = new BasicDBObject()
query.append("cat", "district")
query.append("city", city)
query.append("visible", 0)
DBObject sdq = new BasicDBObject();
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
query.append('$sd', sdq)
return DbUtils.seaweed.find(query).toList()
} catch (Exception e) {
e.printStackTrace()
}
return 0
}
static def findThirdPartyPrice(def city, def region, def name, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
}
return map
}
static def findGXDPrice(def city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("source", "GXD")
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
def price = null
for(def d : dbcursor){
price = d.price
}
return price
}
static def findTownRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
static def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
def fjwcity = null
if(d.level == "城市"){
fjwcity = RegionData.formatCity(d.city)
if(!fjwcity){
if(d.city == "市辖区"){
fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
d.city = d.province
}
}
}
}else if(d.level == "区域"){
if(d.administer == "省直辖县级行政区划"){
fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
d.city = d.district
}
}
}
if(fjwcity){
retMap.put(fjwcity, d)
}else{
println d.toString() +" 格式化不成功!"
}
}
return retMap;
}
static def queryDistNameCity(){
def q = new BasicDBObject()
q.put("level","区域")
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
if(d.city == "省直辖县级行政区划"){
continue
}else if(d.city == "市辖区"){
def fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}else{
def fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}
}
return retMap;
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
def rets = []
def ids = []
for(def d : dbsour){
rets << d
}
println "[${city}] 小区量:"+rets.size()
return rets;
}
}
@@ -24,16 +24,16 @@ class LoudongDataCoverBatch extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
public static final def EMAILS = [
"duxiaoyu@fangjia.com",
// "duxiaoyu@fangjia.com",
// "ruanyadong@fangjia.com",
// "xiamin@fangjia.com",
// "zhutianli@fangjia.com",
"joe@fangjia.com",
"yukai@fangjia.com",
"jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com",
"shijing@fangjia.com",
"xiaohai@fangjia.com",
// "joe@fangjia.com",
// "yukai@fangjia.com",
// "jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com"
// "shijing@fangjia.com",
// "xiaohai@fangjia.com",
]
static def FIELDS_MAP = [
@@ -27,8 +27,8 @@ class ApiDailyCount extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
def static emails = [
'xuqi@fangjia.com',
'zongzhimin@fangjia.com'
'wangwenyan@fangjia.com',
'gaojuan@fangjia.com'
]
static void main(String[] args){
@@ -0,0 +1,97 @@
package com.fangjia.script.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
@BatchOwner(Watcher.GaoJuan)
class DistrictFacilities2Batch extends BaseBatch {
public DBCollection onlineColl = MongoUtils.getCollectionByName("fangjia", "district_arround_poi", "mongo.db.fangjia.com")
public static void main(String[] args) {
//小区周边数据更新
this.runBatch(new DistrictFacilities2Batch(), args);
}
@Override
public void process(Object args) {
def cities = CommonDatautils.findAllCity()
for (def city : cities) {
def ret = CommonDatautils.getAllDistricts("district", city)
def index = 0;
for (def dbo : ret) {
index++;
println index + "/" + ret.size() + " " + dbo.city + " " + dbo.name
String region = dbo.get('region')
String districtName = dbo.get('name')
if (!dbo.lat2 && !dbo.lng2) {
continue;
}
def poms = findPoms(dbo.lat2 / 10000000000.0, dbo.lng2 / 10000000000.0);
DBObject object = new BasicDBObject();
object.put("hospital", unique(processType(poms.get('医院'), 1000, null, ['宠物', '动物', '口腔', '牙科', '美容', '齿科', '眼科', '号楼', '住院部', 'Pure', 'Tokushinkai', '假牙制作', '镶牙', '咪', '喵'])))
object.put("shop", processType(poms.get('超市'), 1000, ['便民商店', '便利店'], ['杂货店', '烟酒', '爱鑫', '超市便利店']))
object.put("university", processType(poms.get('学校'), 1000, ['大学'], ['食堂', '办公室', '德威', '早教', '夕阳红', '公司']))
object.put("middle_school", processType(poms.get('学校'), 1000, ['中学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
object.put("primary_school", processType(poms.get('学校'), 1000, ['小学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
object.put("kindergarten", processType(poms.get('学校'), 1000, ['幼儿园', '托儿所'], ['食堂', '亲子', '办公室', '夕阳红', '公司']))
object.put("food_shop", processType(poms.get('餐饮'), 500, ['餐饮服务']))
def market = [];
if (poms.get('商场')) {
market = market + poms.get('商场');
}
if (poms.get('超市')) {
market = market + poms.get('超市');
}
object.put("supermarket", processType(market, 1000, ['购物中心', "超市", "好又多"], ['公司', '村村寨寨', '伍缘折扣', '杂货店']))
object.put("bank", processType(poms.get('银行'), 1000, ['银行'], ['自动提款机']))
object.put("atm", processType(poms.get('ATM'), 1000, ['自动提款机', "atm", "ATM"]))
updateToDistrictFacilitys(city, region, districtName, object)
updateToDistrictFacilitysScore(city, region, districtName)
}
}
}
def findPoms(def lat, def lng) {
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
DBObject q = new BasicDBObject()
q.put('lat', new BasicDBObject(QueryOperators.LTE, map.maxLat).append(QueryOperators.GTE, map.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, map.maxLng).append(QueryOperators.GTE, map.minLng));
def cursor = POI_COLL.find(q);
def resMap = [:];
for (def dbo : cursor) {
def keyword = dbo.keyword;
def details = resMap.get(keyword);
if (!details) {
details = [];
resMap.put(keyword, details);
}
def dMap = [:];
dMap.name = dbo.name;
dMap.lat = dbo.lat;
dMap.lng = dbo.lng;
dMap.distance = com.fangjia.script.util.GeoUtils.calculateDistance(lat, lng, dbo.lat, dbo.lng);
details << dMap;
}
resMap.each { k, v ->
println "[${k}] " + v;
}
return resMap;
}
}
@@ -75,7 +75,7 @@ class DistrictHouseBatch extends BaseBatch {
sellQuery.put('$facet', facet)
sellQuery.put('city', city)
sellQuery.put('cat',cat)
sellQuery.put('visible',0)
// sellQuery.put('visible',0)
DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0)
List<FacetResult> sellFacetResult = sellDSCursor.getFacetResults();
if(sellFacetResult){
@@ -82,7 +82,7 @@ class LngLatEachWeek extends BaseBatch {
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href)
jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href)
jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href)
// jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href)
}
public static Map<String, Object> getLngLatByAddress(String city, String address){
@@ -1,5 +1,6 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
@@ -38,9 +39,8 @@ class AmapAddressUpdate {
static main(args) {
def task = new AmapAddressUpdate()
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
def addr = task.formatAddress("福州", "北三环路201号")
println addr
def path = "d:\\数据\\小区\\重庆小区地址与高德地图V1.xlsx"
// task.run(path)
// task.similarMatch(path)
// task.addressRun(path)
@@ -50,11 +50,15 @@ class AmapAddressUpdate {
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
// task.updateData(path)
// task.updateLocation(path)//更新经纬度
task.matchNoSameRegionDist()
}
def run(path){
def citys = ["福州"]//CommonDatautils.findAllCity()
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"
def citys = ["重庆"]//CommonDatautils.findAllCity()
def allrets = []
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度","得分","匹配类型","小区经纬度与高德距离","操作","来源","地址与小区经纬度距离","备注","经纬度是否修改","高德类型"]
for (def city : citys) {
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
def rets = findAmapDetail(disticts)
@@ -63,6 +67,17 @@ class AmapAddressUpdate {
ExcelUtils.write(path, allrets, "数据")
}
def matchNoSameRegionDist(){
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
for (def city : citys) {
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address","estate_type2", "addr_alias","lat", "lng"])
def rets = findDiffRegionAmapDetail(disticts)
ExcelUtils.write("D:\\数据\\小区\\区域有问题小区.xlsx", rets, city)
}
}
def amapRun(){
def city = "福州市"
def datas = findAllAmapByCity(city)
@@ -177,7 +192,15 @@ class AmapAddressUpdate {
row << formatAddress(d.city, dbo.address)
row << dbo.lng
row << dbo.lat
row << 1
row << dbo.match_type
if(d.lng && dbo.lng){
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
row << distance
}else{
row << ""
}
}else{
row << ""
row << ""
@@ -186,6 +209,8 @@ class AmapAddressUpdate {
row << ""
row << ""
row << ""
row << ""
row << ""
}
println totalCount.get() +' '+row
rets << row
@@ -201,6 +226,75 @@ class AmapAddressUpdate {
return rets
}
def findDiffRegionAmapDetail(districts){
def rets = Collections.synchronizedList(new ArrayList());
rets << ["城市","区域","名称","地址","物业类型","地图"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dbos = getAmapLocations(d.city+"市", d.name)
if(!dbos){
def newName = d.name
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
}
dbos = getAmapLocations(d.city+"市", newName)
}
if(dbos){
def sameFlag = false
for(def dbo : dbos){
def newRegion = RegionData.formatRegion(d.city, dbo.region)
if(d.region == newRegion){
sameFlag = true
continue
}
}
if(!sameFlag){
def row = []
row << d.city
row << d.region
row << d.name
row << d.address
row << d.estate_type2
def str = ""
for(def dbo : dbos){
def distance = null
if(d.lng && dbo.lng){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
}
str = str + dbo.region+"-"+dbo.name+"-"+dbo.address+"-"+dbo.type+"-"+distance+"\n"
}
row << str
rets << row
}
}
println totalCount.get()+" "+d.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def formatAddress(def city, def address){
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
def address_str = null
@@ -267,6 +361,38 @@ class AmapAddressUpdate {
}
def getAmapLocations(city, name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and type like '%住宅%'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def list = []
while (rs.next()) {
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
def mapName = rs.getString("name").replace("·","")
if(mapName == name){
def map = [:]
map.name = rs.getString("name")
map.address = rs.getString("address")
map.formatAddress = AddressUtil.formatAddress(city, map.address)
map.region = rs.getString("adname")
map.type = rs.getString("type")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
list << map
}
}
return list;
}
@@ -451,9 +577,11 @@ class AmapAddressUpdate {
}
def updateLocation(path){
ExcelUtils.multiThreadingModify(path, { row ->
def index = 0
ExcelUtils.modify(path, { row ->
if(row['R'] && row['R']=="是"){
index++
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
if(dist1){
def udata = new BasicDBObject()
@@ -514,8 +642,9 @@ class AmapAddressUpdate {
row['J'] = dbo.lng
row['K'] = dbo.lat
row['L'] = dbo.score
row['M'] = dbo.distance
row['S'] = dbo.type
row['N'] = dbo.distance
row['M'] = dbo.match_type
row['T'] = dbo.type
}
}
}
@@ -539,7 +668,7 @@ class AmapAddressUpdate {
def mapName = rs.getString("name").replace("·","")
println name +" "+ mapName
if(rs.getString("adname").contains(region)){
if(rs.getString("adname") && rs.getString("adname").contains(region)){
map.name = rs.getString("name")
map.address = rs.getString("address")
@@ -97,7 +97,7 @@ class DistrictInfoExportBatch extends BaseBatch{
saveOrUpdateFzDist(districts)
if(day == 15){
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), endMDate);
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), null);
remarkRemoveDist(removeDists)
}else{
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, startMDate, endMDate);
@@ -201,6 +201,9 @@ class DistrictInfoExportBatch extends BaseBatch{
def udate = new BasicDBObject()
udate.put("status",1)
udate.put("m_date", rd.m_date)
udate.put("region", rd.region)
udate.put("name", rd.name)
udate.put("estate_type2", rd.estate_type2)
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
}
}
@@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createNewDatas("福州")
saveToDB(ret)
def results = selectLastResultDatas(ret);
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
}
def sb = appendResult(headers, ret)
def sb = appendResult(headers, results)
DataHandleUtils.outPutFile(fileName, sb, charset);
FzFtpUtils.ftpUpload(fileName)
//生成加密数据文件并上传
FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset);
}
def selectLastResultDatas(def rets){
def rows = []
for(def r : rets){
rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]]
}
return rows;
}
def appendResult(def headers, def ret){
@@ -121,6 +131,7 @@ class DistrictPriceExportBatch extends BaseBatch{
for(def rMap : districts){
def curPrice = 0
def dbo = null
if(curPriceMap.containsKey(rMap.get("_id").toString())){
dbo = curPriceMap.get(rMap.get("_id").toString())
curPrice = dbo.avg_price
@@ -147,6 +158,29 @@ class DistrictPriceExportBatch extends BaseBatch{
m << 0
}
m << curDate.toInteger();
if(curPrice && dbo){
if(dbo.value_type == 1){
m << "指导价"
}else if(dbo.value_type == 2){
m << "自身样本"
}else if(dbo.value_type == 3){
m << "第三方"
}else if(dbo.value_type == 4){
m << "交易数据"
}else if(dbo.value_type == 5){
m << "周边楼盘"
}else if(dbo.value_type == 6){
m << "推导"
}else if(dbo.value_type == 9){
m << "神经网络"
}else if(dbo.value_type == 10){
m << "无方式"
}else{
m << ""
}
}else{
m << "无"
}
result << m
}
return result
@@ -300,6 +334,7 @@ class DistrictPriceExportBatch extends BaseBatch{
//按comm_id修改,小区名重新赋值,防止区域和小区名修改
udata.put("region", r[3])
udata.put("name", r[4])
udata.put("value_type", r[7])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def o = new BasicDBObject()
@@ -312,6 +347,7 @@ class DistrictPriceExportBatch extends BaseBatch{
o.put("avg_price", r[5].toInteger())
o.put("month", r[6].toInteger())
o.put("c_date", new Date())
o.put("value_type", r[7])
FZ_DISTRICT_STAT_COLL.save(o)
}
}
@@ -32,6 +32,8 @@ class DistrictSellStatBatch extends BaseBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static void main(String[] args) {
runBatch(new DistrictSellStatBatch(), args);
@@ -88,18 +90,21 @@ class DistrictSellStatBatch extends BaseBatch {
}
}else{
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0)
// q.put("name", "翠园楼")
if(day == 2){
districts = DbUtils.seaweed.find(q, keys).batchSize(2000)
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
}else{
q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
districts = DbUtils.seaweed.find(q, keys).batchSize(2000)
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
}
}
println index+" "+city +" "+districts.size()+" "+ (new Date()).toString()
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -1);
def endDate = DateUtils.getLastDayOfMonth(new Date(), -1);
if(districts.size()>0){
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//ÉϸöÔÂÐ¡Çø¾ù¼Û
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
@@ -135,11 +140,12 @@ class DistrictSellStatBatch extends BaseBatch {
def rets = new CopyOnWriteArrayList()
districts.each{dist->
// fixedThreadPool.execute(new Runnable() {
// void run() {
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name
def weekendPriceObj = weekendPriceMap.get(key)
if(weekendPriceObj){
def curPriceObj = curMonthlyPriceMap.get(key)
@@ -151,24 +157,19 @@ class DistrictSellStatBatch extends BaseBatch {
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
}
def curPrice = null;
if(curPriceObj){
curPrice = curPriceObj.avg_price
}else{
curPrice = weekendPriceObj.avg_price
}
def curPrice = weekendPriceObj.avg_price
if(weekendPriceObj.value_type!=1){
if(dist.city == "福州"){
// if(Math.abs(mom)*100>3){
// def randMarkup = nextDouble(0, 3)
// if(mom < 0){
// mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
// }else{
// mom = DataUtil.roundDouble(randMarkup/100, 4)
// }
// curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
// }
if(Math.abs(mom)*100>3){
def randMarkup = nextDouble(0, 3)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
}else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
}else{
if(Math.abs(mom)*100>10){
def randMarkup = nextDouble(0, 10)
@@ -197,7 +198,9 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("commid", dist._id.toString())
data.put("mom", mom)
data.put("yoy", yoy)
data.put("avg_price", curPrice)
data.put("avg_price",curPrice)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
}
@@ -209,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("estate_type", dist.estate_type2)
data.put("date", curMonth)
data.put("avg_price", curPrice)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
data.put("m_date", new Date())
data.put("commid", dist._id.toString())
@@ -224,10 +228,10 @@ class DistrictSellStatBatch extends BaseBatch {
} finally {
latch.countDown()
}
// };
// })
};
})
}
// latch.await()
latch.await()
}
@@ -245,11 +249,12 @@ class DistrictSellStatBatch extends BaseBatch {
}
def findPriceMap(city, weekend){
def findPriceMap(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DISTRICT_STAT.find(q)
// q.put("district_name","金福花园")
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
def priceMap = [:]
for(def d : dbcursor){
if(city == "福州"){
@@ -257,7 +262,9 @@ class DistrictSellStatBatch extends BaseBatch {
d.avg_price = d.avg_price_2b
}
}
priceMap.put(d.region +"##"+d.district_name, d)
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
priceMap.put(d.region +"##"+d.district_name, d)
}
}
return priceMap
}
@@ -22,6 +22,10 @@ class FzDataResultStat {
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
@@ -51,59 +55,67 @@ class FzDataResultStat {
})
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
def dists = getFzDistricts("福州", ["status"])
def idMaps = [:]
for(def d : dists){
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
}
def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")))
def distkeys = []
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
for(def obj : list1){
def key = obj.city + "##" + obj.region + "##" + obj.name
if(!distkeys.contains(key)){
distkeys << key
}else{
println "重复:"+key
}
map1.put(key, row)
map1.put(key, obj)
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(lastMonth)
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row)
for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name
map2.put(key, obj)
}
def rows = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
map1.each {k,list->
def row = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
def index = 0
map1.each {k,obj1->
def keys = k.split("##")
def lastObj = map2.get(k)
def lastPrice = null
if(lastObj){
lastPrice = lastObj[5]
}
def lastPrice = map2.get(k)
def markup = null
try{
if(list && lastPrice && lastPrice.toInteger()>0){
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){
markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger()
}
}catch(Exception e){
e.printStackTrace()
}
def dist = CommonDatautils.findDistrictById(list[1])
def flag = "有效"
if(dist && dist.status!=0){
flag = "失效"
def flag = "无效"
if(obj1){
flag = idMaps.get(obj1.comm_id)
}
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
def markup_abs = null
if(markup){
markup_abs = Math.abs(markup)
}
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
rows << row
index++
println row
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
@@ -255,4 +267,30 @@ class FzDataResultStat {
}
return files;
}
def findFzDistrictStat(lastMonth){
def q = new BasicDBObject()
q.put("month", lastMonth)
return FZ_DISTRICT_STAT_COLL.find(q).toList()
}
def getFzDistricts(def city, fields){
def q = new BasicDBObject()
q.put("city", city)
q.put("cat", "district")
def rets = []
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
FZ_DIST_COLL.find(q, keys).each{it->
rets << it
}
return rets
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -76,7 +76,7 @@ class ThirdPriceExportBatch extends BaseBatch{
row << dist.name
row << o.s_date
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
def zgf_price = getLastThirdPrice(o.xingfuli_esf_price, curPrice)
row << zgf_price
if(zgf_price){
add = true;
@@ -217,8 +217,8 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.s_date){
retMap.s_date = curDate
}
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
if(!retMap.xingfuli_esf_price && d.xingfuli_esf_price){
retMap.xingfuli_esf_price = d.xingfuli_esf_price
}
if(!retMap.anjuke_esf_price && d.get("anjuke_esf_price")){
retMap.anjuke_esf_price = d.get("anjuke_esf_price")
@@ -3,6 +3,8 @@ package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
@@ -25,7 +27,30 @@ class AddAvgpriceBatch {
static main(args) {
def task = new AddAvgpriceBatch();
// task.run1();
task.test()
// task.test()
// task.sendPriceCommand()
task.sendNewDistPrice()
}
def sendNewDistPrice(){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", "district");
q.put("c_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-11-14","yyyy-MM-dd")))
def dbsour = DbUtils.seaweed.find(q)
for(def d : dbsour){
addPrice(d.city,d.region,d.name, d.estate_type2, DateUtils.parseFromTightByPattern('2023-07-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-19','yyyy-MM-dd'))
}
}
def sendPriceCommand(){
ExcelUtils.multiThreadingModify("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx", { row ->
if(!row['P'] && row['T'] && ["别墅","普通住宅","商住两用"].contains(row['T'])){
//发重算均价命令
println row['K']+" "+row['L']+" "+row['M']
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
}
})
}
def test(){
@@ -1,5 +1,6 @@
package com.fangjia.script.gj
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -18,25 +19,24 @@ class EsSearchUtils {
static def findPois(def city, def region, def name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and adname = '" + region + "'")
sqlBuffer.append(" and name = '" + name + "'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString())
def rets = []
while (rs.next()) {
def m = [:]
m.city = rs.getString("cityname")
m.region = rs.getString("adname")
m.name = rs.getString("name")
m.address = rs.getString("address")
m.type = rs.getString("type")
m.code = rs.getString("typecode")
m.lng = rs.getString("location.lon")
m.lat = rs.getString("location.lat")
rets << m
}
ElasticUtils.executeSqlQuery(sql,{ data->
def m = [:]
m.city = data.cityname
m.region = data.adname
m.name = data.name
m.address = data.address
m.type = data.type
m.code = data.typecode
def location = data.location
m.lng = location.lon
m.lat = location.lat
rets << m
})
return rets;
}
@@ -0,0 +1,53 @@
package com.fangjia.script.gj
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
File file = new File("d:\\Êý¾Ý\\ÃÀÍÅ\\FJW_COMM_20230815.csv");
def rows = [];
file.eachLine{
def d = it.split("@",-1)
if(d[1] && d[1]!='¸£ÖÝ' && d[7] && d[7].isNumber()){
rows << d
}
}
println rows.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rows.size())
for(def i=0;i<rows.size();i++){
final def d = rows[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def q = new BasicDBObject()
q.put("city", d[1])
q.put("region", d[2])
q.put("district_name", d[3])
q.put("date", 202307)
def udata = new BasicDBObject()
udata.put("avg_price", d[7].toString().toDouble().toInteger())
udata.put("m_date", new Date())
DISTRICT_SELL_STAT.update(q, new BasicDBObject('$set', udata), false, true)
println totalCount.get() +" "+d[1] + " "+d[3]
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
@@ -6,6 +6,7 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.CNNMFilter
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.BaiduCoordinateConverter
import com.fangjia.script.util.CosineSimilarAlgorithm
import net.sf.json.JSONObject
import net.sf.json.groovy.JsonSlurper
@@ -337,7 +338,7 @@ class OpenApiUtils {
lat2 = null
lng2 = null
}
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+" "+map.estateType+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
}
// }else{
@@ -373,15 +374,18 @@ class OpenApiUtils {
}
static def searchBaiDuMap(String address, String region, int cityCode) {
static def searchBaiDuMap(String region, String township, String keyword,int cityCode) {
def rets = []
keyword = keyword.replace("居民委员会","").replace("居委会","").replace("居委会","").replace("居委","").replace("村民委员会","").replace("村村委会","村").replace("村委会","村")
def address = keyword
try {
Thread.sleep(500)
long t = System.currentTimeMillis()
// String url = "http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1" +
// "&qt=con&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&sug=0&l=13" +
// "&sug_forward=&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}"
String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
// String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
def content = new HttpRequest().get(url, null, "utf-8")
JSONObject json = JSONObject.fromObject(content)
@@ -389,49 +393,58 @@ class OpenApiUtils {
for (def loc : json.content) {
String name = formatName(address)
if(loc.uid) {
def bean = [:]
JSONObject uidInfo = BaiDuApiUtils.getDetailByUid(loc.uid)
JSONObject uidInfo = null;//BaiDuApiUtils.getDetailByUid(loc.uid)
if(uidInfo != null) {
def baiduRegion = uidInfo.get("result").area
def baiduCity = uidInfo.get("result").city
rets = createData(rets, uidInfo, loc, region, name, address)
}else{
def baiduCity = loc.city_name
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
def newName = formatName(loc.name)
if(baiduRegion.contains(region) && (name== newName || address==newName)){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
Double lng2 = lo.get("lng")
Double lat2 = lo.get("lat")
bean.lng2 = lng2
bean.lat2 = lat2
if(!baiduCity){
baiduCity = loc.history_address
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
}
if(!baiduCity){
baiduCity = loc.area_name
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
}
def baiduRegion = loc.area_name.replace(baiduCity,"")
keyword = keyword.replace("社区","")
def bdName = loc.name.replace("居民委员会","").replace("居委会","").replace("居委","").replace("村民委员会","").replace("村村委会","村").replace("村委会","村").replace("社区","")
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(bdName, keyword);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, keyword);
if(score1 || score2){
def bean = [:]
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
def lng = gmap.get("lng")
def lat = gmap.get("lat")
bean.lng = lng
bean.lat = lat
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
if(loc.admin_info){
bean.township = loc.admin_info.town_name
}
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
}else{
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
rets << bean;
def locArrs = BaiduCoordinateConverter.bdMercator2Latlng(loc.diPointX/100.0, loc.diPointY/100.0)
if(locArrs){
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0], locArrs[1])
bean.lng = gmap.get("lng")
bean.lat = gmap.get("lat")
}
bean.estateType = loc.std_tag
bean.score = score1 + score2
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+loc.name+"-"+ loc.addr.replace("[]","")+" "+bean.estateType+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
if(score1 || score2){
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
}
}
@@ -444,6 +457,71 @@ class OpenApiUtils {
return rets;
}
def static createData(def rets, def uidInfo, def loc, region, name, address){
def bean = [:]
def baiduRegion = uidInfo.get("result").area
def baiduCity = uidInfo.get("result").city
baiduCity = RegionData.formatCity(baiduCity)
def newName = formatName(loc.name)
if(baiduRegion.contains(region) && (name== newName || address==newName)){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
Double lng2 = lo.get("lng")
Double lat2 = lo.get("lat")
bean.lng2 = lng2
bean.lat2 = lat2
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
bean.township = loc.admin_info.town_name
bean.score = 1
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
def lng = gmap.get("lng")
def lat = gmap.get("lat")
bean.lng = lng
bean.lat = lat
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
}
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
}else{
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
rets << bean;
}
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
if(score1 || score2){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
bean.lng2 = lo.get("lng")
bean.lat2 = lo.get("lat")
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
bean.township = loc.admin_info.town_name
def gmap = GeocodeUtils.geoconvBaiduToGoogle(bean.lng2, bean.lat2)
bean.lng = gmap.get("lng")
bean.lat = gmap.get("lat")
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
}
bean.score = score1 + score2
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+bean.estateType+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
}
return rets;
}
def static formatName(String name) {
if(name.indexOf("¡¤") != -1) {
name = name.replaceAll("¡¤", "")
@@ -566,9 +644,70 @@ class OpenApiUtils {
return null;
}
static def jsHouseCase(city, region, name, propertyTypeCode){
def url = "http://localhost:8080/fangjia-open/house/case?city=${city}&district=${region}&buildingName=${name}&propertyTypeCode=${propertyTypeCode}"
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def jsHouseCaseByAddress(address, area){
address = address.replace(" ","").replace("%","")
// def url = "http://open.fangjia.com/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
def url = "http://localhost:8080/fangjia-open/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(area){
url = url +"&strctureArea="+area
}
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def houseValueInformation(city, address, area){
address = address.replace(" ","").replace("%","")
def url = "http://open.fangjia.com/comprehensiveQuery/houseValueInformation?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(area){
url = url +"&size="+area
}
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
print(url)
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
@@ -158,7 +158,6 @@ class PoiLocation {
gmap.lat = DataHandleUtils.getDoubleRound(6, gmap.lat)
}
return location;
}
return null;
}
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-06-01"
static def statDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1),"yyyy-MM-dd")
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -0,0 +1,372 @@
package com.fangjia.script.gj.abc
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class SuzouPocTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new SuzouPocTest()
// task.run()
task.newRun()
}
def newRun(){
println("start:"+new Date().toString())
def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls"
newMatch(filePath)
addEvalutePrice(filePath)
addLation(filePath)
addAroundPrice(filePath)
println("end:"+new Date().toString())
}
def newMatch(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['G'] && !d['J']){
def city = d['B']
def address = d['E']
def name = d['G']
def region = d['C']
if(region && name){
address = region + name
}
city = RegionData.formatCity(city)
if(region){
region = RegionData.formatRegion(city, region)
}
def obj = CommonDatautils.findDistrict(city, region, name.trim(), "district");
if(obj){
d['M'] = obj._id.toString()
d['N'] = obj.get("city")
d['O'] = obj.get("region")
d['P'] = obj.get("name")
if(obj.lng){
d['Q'] = obj.get("lng")/10000000000.0
d['R'] = obj.get("lat")/10000000000.0
}
d['S'] = "直接从库中查询"
}else{
//新的匹配算法
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(!jsonObject && !jsonObject.containsKey("result")){
jsonObject = FangjiaClientUtils.newMatch(city, address, "property", true)
}
if(!jsonObject && !jsonObject.containsKey("result")){
def newAddr = d['E']
if(!d['E'].contains(d['C'])){
newAddr = d['C'] + d['E']
}
jsonObject = FangjiaClientUtils.newMatch(city, newAddr, "property", true)
}
if(!jsonObject.containsKey("result")){
//老匹配
jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
if(jsonObject && jsonObject.credit>0){
d['S'] = "老匹配"
}
}
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['M'] = jsonObject.get("id")
d['N'] = jsonObject.get("city")
d['O'] = jsonObject.get("district")
d['P'] = jsonObject.get("name")
d['Q'] = jsonObject.get("lng")
d['R'] = jsonObject.get("lat")
}
}
}
});
}
def addEvalutePrice(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
def city = d['N']
if(city && !d['J']){
def region = d['O']
def name = d['P']
def area = d['F']
def address = d['E']
def floor = null
if (d['H'] && d['H'].toString().isNumber()) {
floor = d['H'].toString().toDouble().toInteger().toString()
}
def height = null
if (d['J'] && d['J'].toString().isNumber()) {
height = d['J'].toString().toDouble().toInteger().toString()
}
def ret1 = OpenApiClient.addressEvaluation(city, region, name, name, area, floor, height)
if (ret1) {
d['J'] = ret1.totalPrice
d['K'] = ret1.avgPrice
}else{
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
if(priceObj){
d['K'] = priceObj.avg_price_2b
if(area && area.toString().isNumber() && !area.toString().contains("+")){
d['J'] = (int)(area.toString().toDouble()*priceObj.avg_price_2b)
}
}
}
}
})
}
def addLation(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
def city = d['B']
if(city && !d['J']){
def address = d['E']
def map = parserByGaode(city, address)
if(map){
d['Q'] = map.lng
d['R'] = map.lat
d['T'] = "地址反解析经纬度"
}
}
})
}
def parserByGaode(def city, def address){
def json = GaoDeApiUtils.geo(city, address)
println json
if(json && json.status=="1"){
def ret = json.geocodes
if(ret){
def str = ret.get(0).location
def level = ret.get(0).level
if(str && ["门牌号","单元号","道路","道路交叉路口","热点商圈","村庄","兴趣点","门址","住宅区"].contains(level)){
def arrs = str.split(",")
return ["lng":arrs[0], "lat":arrs[1]]
}else{
println address +" "+level
return null
}
}
}
}
def addAroundPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['J'] && d['Q']){
def city = RegionData.formatCity(d['B'])
def dist = new BasicDBObject()
dist.city = city
dist.lng = d['Q'].toString().toDouble()*10000000000L
dist.lat = d['R'].toString().toDouble()*10000000000L
dist.estateType = "普通住宅"//d['H']
if(dist.lat) {
List<DBObject> list = DbUtils.findAroundDistrictByRadius(dist, 1000)
if (list) {
list = sortByDistance(dist, list);
if (list && list.size() > 10) {
list = list[0..9]
}
if (list) {
def prices = []
for (def s : list) {
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if (priceObj) {
prices << priceObj.avg_price_2b
}
}
if (prices) {
def arround_price = (int) (prices.sum() / prices.size())
d['k'] = arround_price
def area = d["F"]
if(area && area.toString().isNumber() && !area.toString().contains("+")){
d['J'] = (int)(area.toString().toDouble()*arround_price)
}
}
}
}
}
}
});
}
def sortByDistance(d, list){
def newList = []
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
newList << nb
}
//排序
Collections.sort(newList, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return newList;
}
def run(){
def path = "D:\\数据\\江苏农行\\江苏农1062条(总).xlsx"
addressEvalute(path)
// addFjwTobPrice(path)
// addThirdPrice(path)
// margeExcel(path)
}
def addressEvalute(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['O'] && d['H']){
def city = d['D']
def region = d['E']
def address = d['G']
def name= d['H']
def area = d['I']
def floor = null
if( d['J'] && d['J'].toString().isNumber()){
floor = d['J'].toString().toDouble().toInteger().toString()
}
def height = null
if(d['K'] && d['K'].toString().isNumber()){
height = d['K'].toString().toDouble().toInteger().toString()
}
def ret1 = OpenApiClient.addressEvaluation(city, region, address, name, area, floor, height)
if(ret1){
d['O']= ret1.city
d['P'] = ret1.district
d['Q'] = ret1.name
d['R'] = ret1.totalPrice
d['S'] = ret1.avgPrice
}
def ret2 = OpenApiClient.addressEvaluation(city, region, address, name, area, null, null)
if(ret2){
d['T'] = ret2.totalPrice
d['U'] = ret2.avgPrice
}
}
})
}
def addFjwTobPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['O']){
def priceObj = CommonDatautils.findDistrictPrice(d['O'], d['P'], d['Q'])
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
}
})
}
def addThirdPrice(path) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { d ->
if (d['O'] && !d['V']) {
def dist = CommonDatautils.findDistrict(d['O'], d['P'], d['Q'], "district")
if (dist) {
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310)
def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name)
if (priceMap) {
if (priceMap.containsKey("lianjia_esf_price")) {
d['X'] = priceMap.get("lianjia_esf_price")
}
if (priceMap.containsKey("beike_esf_price")) {
d['Y'] = priceMap.get("beike_esf_price")
}
if (priceMap.containsKey("anjuke_esf_price")) {
d['Z'] = priceMap.get("anjuke_esf_price")
}
}
def cityId = cityCodeDataMap.get(dist.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
if (map) {
d['AA'] = map.avg_price
}
def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01')
if (tradePrice) {
d['AB'] = tradePrice.toInteger()
}
}
}
})
}
def margeExcel(path){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\(合并)江苏农匹配不上149条-.xlsx", 0, 1)
def map = [:]
for(def d : datas){
map.put(d['G'], d['O'])
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农1062条.xlsx",{d->
def commId = map.get(d['G'])
def dist = CommonDatautils.findDistrictById(commId)
if(dist){
d['O'] = dist.city
d['P'] = dist.region
d['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
}
})
}
}
@@ -230,12 +230,46 @@ class CcbCodeUtils {
}
return "11"
}
//室内结构
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
static def getHouseStructureCode(){
return null
}
//室内结构
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
static def getRoomStructureCode(room_structure){
if(room_structure){
if(room_structure.contains("平层")){
return "11"
}else if(room_structure.contains("错层")){
return "12"
}else if(room_structure.contains("复式")){
return "13"
}else if(room_structure.contains("阁楼")){
return "14"
}else if(room_structure.contains("团结户")){
return "15"
}else if(room_structure.contains("老式房")){
return "16"
}else if(room_structure.contains("地下室")){
return "17"
}else if(room_structure.contains("带阁楼")){
return "21"
}else if(room_structure.contains("带车库")){
return "22"
}else if(room_structure.contains("带车位")){
return "23"
}else if(room_structure.contains("带地下室")){
return "24"
}else if(room_structure.contains("带储藏室")){
return "25"
}else if(room_structure.contains("跃层")){
return "18"
}
}
return null;
}
//租赁方式 整租:0 合租:1 转租:2
static def getRentTypeCode(def rentType){
if(rentType){
@@ -90,4 +90,53 @@ class FileUtils {
}
return ret;
}
static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
if(headers){
sb.append(headers.join(splitChar))
sb.append("\n");
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), splitChar))
sb.append("\n");
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -0,0 +1,317 @@
package com.fangjia.script.gj.chongqin
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 重庆poc测试-2023版
*/
class PocTest2023 {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
static main(args) {
def task = new PocTest2023();
task.run();
}
def run(){
// addEvaluationPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试2.xlsx")
// addressEvalute("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx")
// structure("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx") //九级地址拆分
// addThirdPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx")
// baseInfo("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx")
// baseInfoByName("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx")
distScore("D:\\数据\\重庆农商行2023\\楼盘评级样本.xlsx")
}
def addEvaluationPrice(path){
ExcelUtils.multiThreadingModify(path,{ d->
if(d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['N']= ret1.city
d['O'] = ret1.district
d['P'] = ret1.name
d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0)
d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0)
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['Q'] = dist._id.toString()
}
}
}
})
}
def addressEvalute(path){
ExcelUtils.multiThreadingModify(path,{ d->
if(!d['N'] && d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['N']= ret1.city
d['O'] = ret1.district
d['P'] = ret1.name
d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0)
d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0)
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['Q'] = dist._id.toString()
}
}
//新的匹配算法
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(jsonObject != null && jsonObject.credit>0) {
d['R'] = jsonObject.get("city")
d['S'] = jsonObject.get("district")
d['T'] = jsonObject.get("name")
d['U'] = jsonObject.get("id")
}
if( d['Q'] && d['U'] && d['U']==d['Q']){
d['V'] = "T"
}else{
d['V'] = "F"
}
}
})
}
def structure(inputfile){
ExcelUtils.multiThreadingModify(inputfile,{ d->
if(d['E']){
def jsonObject = FangjiaClientUtils.newStructureByLevel9(d['B'], d['E'])
if(jsonObject != null && jsonObject.msgCode==200 && jsonObject.result) {
def ret = jsonObject.result
def str = ""
if(ret.township){
str = ret.township
}
if(ret.street){
str = str + ret.street
}
d['H'] = str
d['I'] = ret.name
d['J'] = ret.buildingNumber
d['K'] = ret.unitNumber
d['L'] = ret.floor
d['M'] = ret.roomNumber
}
}
})
}
// def addAuthor(inputfile){
// def authors = ["曹","张", "陈", "许", "金玉"]
// ExcelUtils.modify(inputfile, { d ->
// if(d['V'] && d['V'] == "F"){
// for(def name : authors){
// d['W'] = name
// }
// }
// });
// }
def baseInfo(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { d ->
if(d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
println address
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['R']= ret1.city
d['S'] = ret1.district
d['F'] = ret1.name
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['T'] = dist._id.toString()
d['G'] = dist.estate_company
d['H'] = dist.developer
d['I'] = dist.cubage_percent
d['J'] = dist.virescence_percent
d['K'] = dist.all_area
d['L'] = dist.loudong_count
d['M'] = dist.household_count
d['N'] = dist.estate_charge
if(dist.open_date){
d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd")
}
if(dist.is_elevator){
d['P'] = "有"
}else{
d['P'] = "无"
}
def newCity = RegionData.formatCity(city)
def obj = findPaiMai(newCity, address)
if(obj){
d['Q'] = "是"
}
}
}
}
})
}
def findPaiMai(city, address){
def q = new BasicDBObject()
q.put("city", city)
def or = new BasicDBList()
or.add(new BasicDBObject("title", new BasicDBObject('$regex', address)))
or.add(new BasicDBObject("address", new BasicDBObject('$regex', address)))
q.append('$or', or)
def obj = paimai43_COLL.findOne(q)
return obj
}
def baseInfoByName(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { d ->
if(d['F']) {
def city = d['B']
def address = d['E']
def dist = CommonDatautils.findDistrict(d['R'], d['S'], d['F'], "district")
if (dist) {
d['T'] = dist._id.toString()
d['G'] = dist.estate_company
d['H'] = dist.developer
d['I'] = dist.cubage_percent
d['J'] = dist.virescence_percent
d['K'] = dist.all_area
d['L'] = dist.loudong_count
d['M'] = dist.household_count
d['N'] = dist.estate_charge
if (dist.open_date) {
d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd")
}
if (dist.is_elevator) {
d['P'] = "有"
} else {
d['P'] = "无"
}
def newCity = RegionData.formatCity(city)
def obj = findPaiMai(newCity, address)
if (obj) {
d['Q'] = "是"
}
}
}
})
}
def distScore(inputfile) {
ExcelUtils.multiThreadingModify(inputfile, { d ->
if (d['F']) {
def obj = getDistrictRank(d['R'], d['S'], d['F'])
if(obj){
def score = obj.city_rank*100/obj.city_rank_total
score = DataUtil.getRound(score)
d['G'] = 100-score
d['H'] = obj.city_rank
d['I'] = obj.city_rank_total
d['J'] = obj.avg_price
}
}
})
}
def getDistrictRank(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name",name)
return DISTRICT_ANALY_DATA_COLL.findOne(q)
}
def addThirdPrice(path) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { d ->
if (d['P']) {
def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
if (dist) {
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
// if(priceObj){
// d['V'] = priceObj.avg_price
// d['W'] = priceObj.avg_price_2b
// }
def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310)
def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name)
if (priceMap) {
if (priceMap.containsKey("lianjia_esf_price")) {
d['X'] = priceMap.get("lianjia_esf_price")
}
if (priceMap.containsKey("beike_esf_price")) {
d['Y'] = priceMap.get("beike_esf_price")
}
if (priceMap.containsKey("anjuke_esf_price")) {
d['Z'] = priceMap.get("anjuke_esf_price")
}
}
def cityId = cityCodeDataMap.get(dist.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
if (map) {
d['AA'] = map.avg_price
}
def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01')
if (tradePrice) {
d['AB'] = tradePrice.toInteger()
}
}
}
})
}
}
@@ -1,19 +1,19 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.city
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class CityStatExport {
@@ -28,8 +28,41 @@ class CityStatExport {
def task = new CityStatExport();
// task.run()
// task.run1()
task.regionStat()
// task.regionStat()
// task.stat()
// task.houseStat()
task.addFieldStat()
}
def addFieldStat(){
def fileName = "D:\\数据\\fjw.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
if(!d['B']){
def city = RegionData.formatCity(d['A'])
if(city){
def cityInfoDto = CommonDatautils.findCityInfo(city)
if(cityInfoDto){
d['B'] = cityInfoDto.existing_district_count
def detailMap = cityInfoDto.district_detail
// d['D'] = detailMap.address.toDouble()*100
// d['E'] = 100
d['D'] = cityInfoDto.price_coverage.toDouble()*100
}
}
}
},1)
}
def houseStat(){
def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
// def houseCount = findHouseCount(d['F'], 20230201)
// d['H'] = houseCount
d['J'] = findHouse(d['F'])
})
}
def stat(){
@@ -47,7 +80,7 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity();
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"]
def ret = []
ret << ["城市","小区总量"]
// ret << ["城市","楼栋总量", "房号总量"]
@@ -78,7 +111,7 @@ class CityStatExport {
ret << m
println m
}
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3")
}
@@ -124,7 +157,7 @@ class CityStatExport {
def findHouseCount(def city, def startDate){
def total = 0
try{
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
@@ -136,6 +169,18 @@ class CityStatExport {
return total
}
def findHouse(city) {
int startDate = 20230201
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
.must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses.size()
}
def findLoudongCount(String city) {
return LOUDONG.count(new BasicDBObject('city', city))
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.commStat
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
@@ -27,7 +28,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
//for(def city : citys){
@@ -308,21 +309,65 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
//区域级统计
def city = "合肥"
def regions = CommonDatautils.getCityRegions(city)
def rows = []
for(def region : regions){
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
def alias_count = 0
def addr_alias_count = 0
for(def r : districts){
if(r.alias && r.alias.size()>0){
alias_count ++
//def city = "合肥"
//def regions = CommonDatautils.getCityRegions(city)
//def rows = []
//for(def region : regions){
// def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
// def alias_count = 0
// def addr_alias_count = 0
// for(def r : districts){
// if(r.alias && r.alias.size()>0){
// alias_count ++
// }
// if(r.addr_alias && r.addr_alias.size()>0){
// addr_alias_count++
// }
// }
// rows << [region, districts.size(), alias_count, addr_alias_count]
//}
//ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\区县楼盘量统计.xlsx",{d->
if(d['C'] && !d['I']){
def city = RegionData.formatCity(d['C'])
if(!city){
city = RegionData.formatCity(d['B'])
}
if(r.addr_alias && r.addr_alias.size()>0){
addr_alias_count++
if(city && d['D']){
def region = RegionData.formatRegion(city, d['D'])
if(region){
def count = CommonDatautils.findRegionDistrictCount("district", city, region)
println city +" "+ region+" "+count
d['I'] = city
d['J'] = region
d['K'] = count
}
// def dists = CommonDatautils.findAllDistricts(d['C'], "district", ["city","region","name"])
//d['B'] = dists.size()
}
}
rows << [region, districts.size(), alias_count, addr_alias_count]
}
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
// def q = new BasicDBObject()
// q.put("city",d['B'])
// q.put("district","")
// def dbo = mca_administrative_division_coll.findOne(q)
// if(!dbo){
// q.put("city", "省直辖县级行政区划")
// q.put("district", d['B'])
// dbo = mca_administrative_division_coll.findOne(q)
// }
// if(dbo){
// d['A']=dbo.province
// }
})
//def citys = ["北京","上海","重庆","天津","石家庄","唐山","秦皇岛","邯郸","邢台","保定","张家口","承德","沧州","廊坊","衡水","太原","大同","阳泉","长治","晋城","朔州","晋中","运城","忻州","临汾","吕梁","呼和浩特","包头","乌海","赤峰","通辽","鄂尔多斯","呼伦贝尔","巴彦淖尔","乌兰察布","兴安盟","锡林郭勒盟","阿拉善盟","沈阳","大连","鞍山","抚顺","本溪","丹东","锦州","营口","阜新","辽阳","盘锦","铁岭","朝阳","葫芦岛","长春","吉林","四平","辽源","通化","白山","松原","白城","延边","哈尔滨","齐齐哈尔","鸡西","鹤岗","双鸭山","大庆","伊春","佳木斯","七台河","牡丹江","黑河","绥化","大兴安岭","南京","无锡","徐州","常州","苏州","南通","连云港","淮安","盐城","扬州","镇江","泰州","宿迁","杭州","宁波","温州","嘉兴","湖州","绍兴","金华","衢州","舟山","台州","丽水","合肥","芜湖","蚌埠","淮南","马鞍山","淮北","铜陵","安庆","黄山","滁州","阜阳","宿州","六安","亳州","池州","宣城","福州","厦门","莆田","三明","泉州","漳州","南平","龙岩","宁德","南昌","景德镇","萍乡","九江","新余","鹰潭","赣州","吉安","宜春","抚州","上饶","济南","青岛","淄博","枣庄","东营","烟台","潍坊","济宁","泰安","威海","日照","临沂","德州","聊城","滨州","菏泽","郑州","开封","洛阳","平顶山","安阳","鹤壁","新乡","焦作","濮阳","许昌","漯河","三门峡","南阳","商丘","信阳","周口","驻马店","武汉","黄石","十堰","宜昌","襄阳","鄂州","荆门","孝感","荆州","黄冈","咸宁","随州","恩施","长沙","株洲","湘潭","衡阳","邵阳","岳阳","常德","张家界","益阳","郴州","永州","怀化","娄底","湘西","广州","韶关","深圳","珠海","汕头","佛山","江门","湛江","茂名","肇庆","惠州","梅州","汕尾","河源","阳江","清远","东莞","中山","潮州","揭阳","云浮","南宁","柳州","桂林","梧州","北海","防城港","钦州","贵港","玉林","百色","贺州","河池","来宾","崇左","海口","三亚","三沙","儋州","成都","自贡","攀枝花","泸州","德阳","绵阳","广元","遂宁","内江","乐山","南充","眉山","宜宾","广安","达州","雅安","巴中","资阳","阿坝州","甘孜州","凉山州","贵阳","六盘水","遵义","安顺","毕节","铜仁","黔西南","黔东南","黔南","昆明","曲靖","玉溪","保山","昭通","丽江","普洱","临沧","楚雄","红河","文山州","西双版纳州","大理","德宏州","怒江州","迪庆州","拉萨","日喀则","昌都","林芝","山南","那曲","阿里","西安","铜川","宝鸡","咸阳","渭南","延安","汉中","榆林","安康","商洛","兰州","嘉峪关","金昌","白银","天水","武威","张掖","平凉","酒泉","庆阳","定西","陇南","临夏","甘南","西宁","海东","海北州","黄南州","海南州","果洛州","玉树州","海西州","银川","石嘴山","吴忠","固原","中卫","乌鲁木齐","克拉玛依","吐鲁番","哈密","昌吉","博尔塔拉","巴音郭楞","阿克苏","克孜勒苏","喀什","和田","伊犁","塔城","阿勒泰","济源","仙桃","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","石河子","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉","昆玉"]
//
//def fjwCitys = CommonDatautils.findAllCity()
//for(def fjw_city : fjwCitys){
// if(!citys.contains(fjw_city)){
// println fjw_city
// }
//}
@@ -14,12 +14,12 @@ class FieldStat {
static main(args) {
def task = new FieldStat();
// task.run()
// task.run1()
task.run1()
// task.run2()
// task.run3()
// task.offircRun()
task.distRun()
task.shopRun()
// task.distRun()
// task.shopRun()
}
def distRun(){
@@ -203,11 +203,15 @@ class FieldStat {
alias_total = alias_total + alias.size()
}
}
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
def precent = 0
if(districts){
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
}
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
}
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
ExcelUtils.write("d:\\数据\\城市别名统计.xlsx", rets, "统计")
// println dist_total
// println alias_total
// println addr_alias_total
@@ -5,24 +5,38 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//
//ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
// def opt = d['J']
// if(opt && opt.contains("别名")){
// def dist = CommonDatautils.findDistrictById(d['H'])
// if(dist){
// def alias = []
// if(dist.alias){
// alias = dist.alias
// }
// if(!alias.contains(d['C'])){
// alias << d['C']
// def udata = new BasicDBObject()
// udata.put("alias", alias)
// DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
// }
//
// }
// }
//})
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
ExcelUtils.modify("D:\\数据\\厦门\\厦门户数(曹和付)-结果.xlsx",{d->
def opt = d['J']
if(opt && opt.contains("别名")){
def dist = CommonDatautils.findDistrictById(d['H'])
if(opt && opt.toString().toDouble().toInteger()==1){
def dist = CommonDatautils.findDistrict('厦门',d['G'],d['H'], "district")
if(dist){
def alias = []
if(dist.alias){
alias = dist.alias
}
if(!alias.contains(d['C'])){
alias << d['C']
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
def udata = new BasicDBObject()
udata.put("household_count", d['F'].toString().toDouble().toInteger())
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
}
})
@@ -5,8 +5,10 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -22,12 +24,177 @@ class ApiTest {
static main(args) {
def task = new ApiTest()
// task.rankRunTest()
// task.similarRunTest()
task.loupanLevelTest()
// task.rankRunTest()//小区排名接口测试
// task.similarRunTest()//相似房源接口测试
// task.loupanLevelTest()//楼盘评级接口测试
// task.jsAbcHouseCastTest()//北交所房源接口测试 小区
// task.arroundFacilities()//北交所周边poi 小区
// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
// task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
//五级地址标准化
// task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
// task.nineLevel("D:\\数据\\地数\\地址九级标准化-测试用例.xlsx")
}
def addrStructure(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
if(jsonObject) {
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['C'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['D'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['E'] = jsonObject.get("district")
}
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
d['F'] = jsonObject.get("township")
}
if(jsonObject.get("address") && jsonObject.get("address")!="null"){
d['G'] = jsonObject.get("address")
}
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
d['H'] = jsonObject.get("provinceCode")
}
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
d['I'] = jsonObject.get("cityCode")
}
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
d['J'] = jsonObject.get("districtCode")
}
}
}
},7);
}
def nineLevel(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def obj = FangjiaClientUtils.formatNinelevel(d['A'], d['B']);
if(obj) {
def jsonObject = obj.get("result")
if(jsonObject){
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['C'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['D'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['E'] = jsonObject.get("district")
}
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
d['F'] = jsonObject.get("township")
}
if(jsonObject.get("formatAddress") && jsonObject.get("formatAddress")!="null"){
d['G'] = jsonObject.get("formatAddress")
}
if(jsonObject.get("community") && jsonObject.get("community")!="null"){
d['H'] = jsonObject.get("community")
}
if(jsonObject.get("name") && jsonObject.get("name")!="null"){
d['I'] = jsonObject.get("name")
}
if(jsonObject.get("street") && jsonObject.get("street")!="null"){
d['J'] = jsonObject.get("street")
}
if(jsonObject.get("streetNumber") && jsonObject.get("streetNumber")!="null"){
d['K'] = jsonObject.get("streetNumber")
}
if(jsonObject.get("buildingNumber") && jsonObject.get("buildingNumber")!="null"){
d['L'] = jsonObject.get("buildingNumber")
}
if(jsonObject.get("unitNumber") && jsonObject.get("unitNumber")!="null"){
d['M'] = jsonObject.get("unitNumber")
}
if(jsonObject.get("roomNumber") && jsonObject.get("roomNumber")!="null"){
d['N'] = jsonObject.get("roomNumber")
}
if(jsonObject.get("floor") && jsonObject.get("floor")!="null"){
d['O'] = jsonObject.get("floor")
}
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
d['P'] = jsonObject.get("provinceCode")
}
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
d['Q'] = jsonObject.get("cityCode")
}
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
d['R'] = jsonObject.get("districtCode")
}
def map = jsonObject.get("addressAssessment")
if(map){
d['S'] = map.get("msg")
}
}
}
}
},0);
}
def addressEvaluate(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def ret1 = OpenApiClient.addressEvaluation(d['A'], null, d['B'], null, d['C'].toString().toDouble().toString(), null, null)
if(ret1){
d['G']= ret1.city
d['H'] = ret1.district
d['I'] = ret1.name
d['J'] = ret1.totalPrice
d['K'] = ret1.avgPrice
}
}
})
}
def addressAvgPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
println "[match] city:"+d['A']+" address:"+d['B']
def jsonObject = FangjiaClientUtils.newMatchPrice(d['A'], d['B'], "property", false)
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
d['G'] = resultMap.get("city")
d['H'] = resultMap.get("district")
d['I'] = resultMap.get("name")
def type = resultMap.get("propertyType")
if(type && type!='null'){
d['J'] = type
}
if(resultMap.get("avgPrice")){
d['K'] = resultMap.get("avgPrice")
}
}
}
})
}
def loupanLevelTest(){
ExcelUtils.modify("D:\\数据\\卓信\\上海 - 副本.xlsx", { d ->
if (d['A']) {
@@ -125,6 +292,265 @@ class ApiTest {
return dbo
}
def jsAbcHouseCastTest(){
def citys = ["无锡"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","estate_type2"])
def rets = getHouseCases(districts)
ExcelUtils.write("D:\\数据\\江苏\\房源样本.xlsx", rets, city)
}
}
def getHouseCases(districts){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
rets << ["城市","区域","名称", "物业类型", "最小案例","最大案例", "案例"]
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def propertyTypeCode = null
if(d.estate_type2=="普通住宅"){
propertyTypeCode = "01"
}else if(d.estate_type2=="别墅"){
propertyTypeCode = "02"
}else if(d.estate_type2=="商住"){
propertyTypeCode = "03"
}
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCase(d.city, d.region, d.name, propertyTypeCode)
if(results){
def row = []
row << d.city
row << d.region
row << d.name
row << d.estate_type2
row << results.minUnitPrice
row << results.maxUnitPrice
def datas = results.datas
if(datas){
def str = ""
for(def r : datas){
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
}
row << str
}
rets << row
}
println totalCount.get()+" "+d.city +" "+d.region+" "+d.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def arroundFacilities(){
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city","region","name","estate_type2"])
// districts = districts[0..1000]
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
rets << ["城市","区域","名称", "重点学校","学校", "公交","地铁","超市","医院","银行"]
for(def i=0;i<districts.size();i++){
final def d = districts[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def row = []
row << d.city
row << d.region
row << d.name
def facilityObj = OpenApiClient.facilitys(d.city, d.region, d.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(result.primarySchool){
row <<result.primarySchool.join(",").replace('"',"")
}else {
row << ""
}
if(result.school){
row << result.school.join(",").replace('"',"")
}else {
row << ""
}
if(result.bus){
row << result.bus.join(",").replace('"',"")
}else{
row << ""
}
if(result.metro){
row << result.metro.join(",").replace('"',"")
}else{
row << ""
}
if(result.supermarket){
row << result.supermarket.join(",").replace('"',"")
}else{
row << ""
}
if(result.hospital){
row << result.hospital.join(",").replace('"',"")
}else{
row << ""
}
if(result.bank){
row << result.bank.join(",").replace('"',"")
}else{
row << ""
}
}
println totalCount.get() + " "+d.region +" "+d.name
rets << row
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
ExcelUtils.write("D:\\数据\\江苏\\南京周边配套.xlsx", rets, "南京")
}
def arroundFacilitiesByAddress(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def facilityObj = OpenApiClient.facilitysByAddress(d['A']+d['B'])
if(facilityObj && facilityObj.result){
def result = facilityObj.result
d['G'] = result.city
d['H'] = result.region
d['I'] = result.districtName
if(result.primarySchool){
d['J'] = result.primarySchool.join(",").replace('"',"")
}else {
d['J'] = ""
}
if(result.school){
d['K'] = result.school.join(",").replace('"',"")
}else {
d['K'] = ""
}
if(result.bus){
d['L'] = result.bus.join(",").replace('"',"")
}else{
d['L'] = ""
}
if(result.metro){
d['M'] = result.metro.join(",").replace('"',"")
}else{
d['M'] = ""
}
if(result.bank){
d['N'] = result.bank.join(",").replace('"',"")
}else{
d['N'] = ""
}
if(result.supermarket){
d['O'] = result.supermarket.join(",").replace('"',"")
}else{
d['O'] = ""
}
if(result.hospital){
d['P'] = result.hospital.join(",").replace('"',"")
}else{
d['P'] = ""
}
}
}
})
}
def jsAbcHouseCastTestByAddress(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B'] && !d['H']){
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCaseByAddress(d['A']+d['B'], d['C'])
if(results){
d['H'] = results.minUnitPrice
d['I'] = results.maxUnitPrice
def datas = results.datas
if(datas){
def str = ""
for(def r : datas){
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
}
d['J'] = str
}
}
}
});
}
def houseValueInformation(path){
ExcelUtils.multiThreadingModify(path,{d->
def results = com.fangjia.script.gj.OpenApiUtils.houseValueInformation(d['A'], d['B'], d['C'])
if(results) {
def addressValue = results.addressValue
if (addressValue) {
d['D'] = addressValue.city + "-" + addressValue.district + "-" + addressValue.name + "-" + addressValue.avgPrice + "-" + addressValue.totalPrice
}
def communityAvgPrice = results.communityAvgPrice
if (communityAvgPrice) {
d['E'] = communityAvgPrice.toString()
}
def surroundingPoi = results.surroundingPoi
if (surroundingPoi) {
d['F'] = surroundingPoi.toString()
}
def house = results.houseCase
if (house) {
d['G'] = house.minUnitPrice
d['H'] = house.maxUnitPrice
def datas = house.datas
if (datas) {
def str = ""
for (def r : datas) {
str = str + r.buildingName + "-" + r.strctureArea + "-" + r.unitPrice + "-" + r.sampleDate.substring(0, 10) + "\n"
}
d['J'] = str
}
}
}
})
}
}
@@ -39,8 +39,10 @@ class DistrictFieldAddBatch extends BaseBatch{
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
//"alias":"alias",
//ownership_type:权属类型 为商品房、经适房、动迁房等
@@ -51,6 +53,9 @@ class DistrictFieldAddBatch extends BaseBatch{
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
"loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
// def fieldsMap = ["household_count":"households"]
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
@@ -67,7 +72,7 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity().reverse()
def citys = CommonDatautils.findAllCity()
def analysRets = []
@@ -88,12 +93,12 @@ class DistrictFieldAddBatch extends BaseBatch{
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
if(totalCount){
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
analysRets = addFieldFromMatch(c, totalCount, analysRets)
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
}
}
println analysRets.toString()
sendStatEmail(headers, analysRets, rh_analysRets)
// sendStatEmail(headers, analysRets, rh_analysRets)
// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
}
@@ -106,11 +111,11 @@ class DistrictFieldAddBatch extends BaseBatch{
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
def updateRets = processFieldsFromMatch(c , matchRets2)
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
//updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
@@ -199,11 +204,13 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def processFieldsFromMatch(c,matchRets){
def processFieldsFromMatch(c,matchRets, esWebSource){
def fields = fieldsMap.keySet() as List
fields << "city"
fields << "estate_type2"
def fieldValueMap = [:]
def rets = []
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
for(def matchDist : matchRets){
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
@@ -212,11 +219,13 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newValues = strandField(c, dist, matchDist)
if(newValues){
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
}
}
// ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource)
return fieldValueMap;
}
@@ -332,7 +341,7 @@ class DistrictFieldAddBatch extends BaseBatch{
def findEsDatas(c, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' ", {r->
list << r
})
return list
@@ -379,6 +388,7 @@ class DistrictFieldAddBatch extends BaseBatch{
try{
totalCount.addAndGet(1)
def ret = matching(data)
// def ret = newMatching(data)
if(ret){
// println totalCount.get() +" "+data.city+" "+data.name+" [Match] "+data.match_region+" "+data.match_name +" "+data.flag+" "+data.credit
rets << ret
@@ -399,7 +409,7 @@ class DistrictFieldAddBatch extends BaseBatch{
def selectRestByCredit(matchRets){
def matchRets1 = []
for(def r : matchRets){
if(r.credit && r.credit>=0.9){
if(r.credit && r.credit>0.9){
matchRets1 << r
}
}
@@ -458,6 +468,58 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newMatching(data){
def name = data.name
if(data.name && data.name.contains("·")){
name = data.name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(data.city, data.region, name)
if(!dist){
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district")
flag = "映射别名匹配"
}
if(!dist) {
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district")
if(rets && rets.size()==1){
flag = "别名匹配"
dist = rets[0]
}
}
if(dist){
data.flag = flag
data.match_id = dist._id.toString()
data.match_region = dist.region
data.match_name = dist.name
data.credit = 0.95
}else{
def jsonObject = FangjiaClientUtils.newMatch(data.city, data.region+name, "property", false)
if(jsonObject!=null && jsonObject.credit>0){
if(data.address && isStandardRoad(data.address)){
jsonObject = FangjiaClientUtils.newMatch(data.city, data.address, "property", false)
}else{
data.flag = "匹配接口-地址"
}
}else{
data.flag = "匹配接口-名称"
}
if(jsonObject != null && jsonObject.credit>0) {
data.match_id = jsonObject.get("id")
data.match_region = jsonObject.get("district")
data.match_name = jsonObject.get("name")
data.credit = jsonObject.get("credit")
}
}
if(data.match_id){
return data
}
return null
}
def updateDB(c, updateRets){
for(def rm : updateRets) {
@@ -114,6 +114,27 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def structure(def city, def address){
def jsonObj = null
def content = null
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
}
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
@@ -172,4 +193,179 @@ class FangjiaClientUtils {
}
}
public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1"
if(floor){
url = url + "&floor="+floor.toString().toDouble().toInteger()
}
if(maxFloor){
url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger()
}
if(room){
url = url + "&room="+room.toString().toDouble().toInteger()
}
if(hall){
url = url + "&hall="+hall.toString().toDouble().toInteger()
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def newMatch(city,address, property, useMap){
def jsonObj = null
def content = null
def url = null;
try {
address = address.replace("{","").replace("\\","")
url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
if(property){
url = url + "&category=${property}"
}
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println "±¨´íµØÖ·£º"+url
e.printStackTrace();
}
return jsonObj;
}
public static def newMatchPrice(city,address, property, useMap){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://open.fangjia.com/addrSvc/avgPrice?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
if(property){
url = url + "&category=${property}"
}
url = url + "&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def newStructureByLevel9(city,address){
def jsonObj = null
def content = null
try {
address = address.replace("{","").replace("\\","")
def url = "http://192.168.0.100:32080/FangJiaAddressService/v2/structure?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def newStandard(address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def formatFivelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def formatNinelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def villageIdentify(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify?address="+URLEncoder.encode(address,"UTF-8")+"&type=2"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{
// strandField(offices)
// }
// addBlock()
// addFieldValue()
// updateErrorField()
export()
test()
// export()
// test()
// addOffice()
fileAddBlock()
}
def export(){
@@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{
})
}
def fileAddBlock(){
def rets = findAllBlockRange()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d->
def location = d['T']
if(location && !d['F']){
def arrs = location.split(",")
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
if(baidu){
def lng2 = baidu.x
def lat2 = baidu.y
def lastObj = null
for (def r : rets) {
def flag = InsidePolygon.isInArea(lng2, lat2, r.points)
if (flag && r.name) {
lastObj = r;
break;
}
}
if (lastObj) {
d['F'] = lastObj.name
}
}
}
})
}
}
@@ -0,0 +1,42 @@
package com.fangjia.script.gj.district
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def city = "常州"
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"])
println districts.size()
def rets = []
rets << ["城市", "区域", "名称", "房价网ToC", "房价网ToB", "近三个月成交均价"]
def index = 0
for(def d : districts){
index++
def ret = []
def dbo = CommonDatautils.queryDistrictPrice(d.city, d.region, d.name, DateUtils.parseFromTightByPattern("2023-09-17","yyyy-MM-dd"))
ret << d.city
ret << d.region
ret << d.name
if(dbo){
ret << dbo.avg_price
ret << dbo.avg_price_2b
}
def map = CcbCommonUtils.findTrades(d, "2023-05-01","2023-09-01")
if(map){
ret << map.avg_price
}
rets << ret
println index
}
ExcelUtils.write("d:\\数据\\江南农商行\\常州小区价格.xlsx", rets, "数据")
@@ -0,0 +1,76 @@
package com.fangjia.script.gj.districtwash
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
class DistrictTypeUpdate {
static main(args) {
def task = new DistrictTypeUpdate();
// task.run()
task.updateType()
}
def updateType(){
ExcelUtils.modify("D:\\数据\\厦门\\物业类型可疑小区(第二批).xlsx",{d->
def flag = d['F']
if(flag && flag.toString().toDouble().toInteger()==2){
def name = d['C']
def type = null
if(name && name.toString().contains("商住")){
type = "商住"
}else{
type = "别墅"
}
def udata = [:]
udata.put("status", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
udata.put("city", d['A'])
udata.put("region", d['B'])
udata.put("name", d['C'])
udata.put("estateType2", type)
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println d['C'] +" "+s
}
})
}
def run(){
def citys = CommonDatautils.findAllCity()
def datas = []
citys.each{c->
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name","estate_type2", "c_date"])
districts.each{d->
def row = null
if(d.name.contains("商住")){
if(d.estate_type2!="商住") {
row = [d.city, d.region, d.name, d.estate_type2, DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd")]
}
}else{
if(d.name.contains("墅")){
if(d.estate_type2!="别墅"){
row = [d.city, d.region, d.name, d.estate_type2, DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd")]
}
}
}
if(row){
datas << row
println row
}
}
}
ExcelUtils.write("d:\\数据\\小区\\物业类型可疑小区.xlsx", datas, "小区")
}
}
@@ -0,0 +1,72 @@
package com.fangjia.script.gj.es
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
def datas = findHouseCount("北京", 20230201, 20230430)
ExcelUtils.write("D:\\数据\\天弘基金.xlsx", datas, "样例")
def findHouseCount(def city, def startDate, def endDate){
def rets = []
try{
// String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='ke.com' order by p_date desc"
def startMdate = DateUtils.parseFromTightByPattern(startDate.toString(),"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
rets << ["城市","区域","板块","名称","面积","单价","总价","室","厅","卫","装修","朝向","发布时间","物业类型","楼层状况","楼高","建筑结构","建筑年代","标题","URL"]
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
// QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(QueryBuilders.termQuery("root", "ke.com"))
.must(date_range).must(area_range)//.must(date_range_mdate)
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses) {
for (int k = 0; k < houses.size(); k++) {
println k
def data = houses[k];
def r = []
r << data.city
r << data.region
r << data.block
r << data.district_name
r << data.area
r << data.avg_price
r << data.total
r << data.room
r << data.hall
r << data.toilet
r << data.fitment
r << data.direction
r << data.p_date
r << data.estate_type
r << data.floor_info
r << data.height
r << data.build_structure
r << data.build_date
r << data.name
r << data.url
rets << r
if(rets.size()>500){
break;
}
}
}
}catch(Exception e){
e.printStackTrace()
}
return rets
}
@@ -25,8 +25,8 @@ class DistrictLocationBatch {
static main(args) {
def task = new DistrictLocationBatch()
// task.run()
task.deleteDist()
task.run()
// task.deleteDist()
}
def deleteDist(){
@@ -28,7 +28,10 @@ class GaoDeMapSearchTest {
// task.mapSearch(path)
// task.baiduMapSearch(path)
// task.checkTrue(path)
task.addFinancialLevel(path)
// task.addFinancialLevel(path)
def results = com.fangjia.script.gj.OpenApiUtils.financialRating("重庆", "凤中路193号")
println results
}
def addFinancialLevel(path){
@@ -0,0 +1,289 @@
package com.fangjia.script.gj.meishubao
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class MsbPocTest {
static main(args) {
def task = new MsbPocTest();
task.run();
}
def run(){
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
// matchOld(inputfile)
// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx")
// test("D:\\数据\\美术宝\\test2.xlsx")
// cityPrice()
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx")
}
def cityPrice(){
def datas = ExcelUtils.read("D:\\数据\\临时\\城市房产价格.xlsx",0,1)
def map = [:]
for(def d : datas){
map.put(d['A'], d)
}
def ccbdatas = ExcelUtils.read("D:\\数据\\临时\\建行城市均价.xlsx",0,1)
def ccbmap = [:]
for(def d : ccbdatas){
ccbmap.put(d['A'], d['B'])
}
ExcelUtils.modify("D:\\数据\\临时\\房价网城市均价.xlsx", { row ->
if(!row['C']){
def d = map.get(row['A'])
if(!d){
d = map.get(row['A'].replace('州',''))
}
if(d){
row['C'] = d[1]
row['D'] = d[2]
}
def d1 = ccbmap.get(row['A'])
if(!d1){
d1 = ccbmap.get(row['A'].replace('州',''))
}
if(d1){
row['E'] = d1
}
}
})
}
def matchOld(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['S'] && row['D']){
def city = row['C']
def region = row['D']
def name = row['F']
def address = row['H']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['S'] = jsonObject.get("id")
row['T'] = jsonObject.get("city")
row['U'] = jsonObject.get("district")
row['V'] = jsonObject.get("name")
row['W'] = jsonObject.get("propertyType")
row['X'] = jsonObject.get("credit")
}
}
})
}
def test(file){
ExcelUtils.multiThreadingModify(file, { row ->
def jsonObject = FangjiaClientUtils.newMatch(null, row['A'], null, true)
println row['A']
if(jsonObject != null && jsonObject.credit>0) {
row['B'] = jsonObject.get("id")
row['C'] = jsonObject.get("city")
row['D'] = jsonObject.get("district")
row['E'] = jsonObject.get("name")
def type = jsonObject.get("propertyType")
if(type && type!='null'){
row['F'] = type
}
row['G'] = jsonObject.get("credit")
}
})
}
def match(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
def city = row['C']
def region = row['D']
def address = row['G'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
if(jsonObject != null && jsonObject.credit>0) {
row['J'] = jsonObject.get("id")
row['K'] = jsonObject.get("city")
row['L'] = jsonObject.get("district")
row['M'] = jsonObject.get("name")
def type = jsonObject.get("propertyType")
if(type && type!='null'){
row['N'] = type
}
row['O'] = jsonObject.get("credit")
}
}
})
}
def mergeMatchPrice(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['L']){
def city = row['C']
def region = row['D']
def name = row['F']
def address = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, "property", false)
if(jsonObject && !jsonObject.containsKey("result")){
jsonObject = FangjiaClientUtils.newMatchPrice(city, address, "property", false)
}
if(jsonObject && !jsonObject.containsKey("result")){
jsonObject = FangjiaClientUtils.newMatchPrice(city, region + name, null, true)
}
if(jsonObject && !jsonObject.containsKey("result")){
jsonObject = FangjiaClientUtils.newMatchPrice(city, address, null, true)
}
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
row['L'] = resultMap.get("city")
row['M'] = resultMap.get("district")
row['N'] = resultMap.get("name")
def type = resultMap.get("propertyType")
if(type && type!='null'){
row['O'] = type
}
if(resultMap.get("avgPrice")){
row['P'] = resultMap.get("avgPrice")
}
}
}
})
}
def addCCBPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['M']){
def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
if(dist){
def d = CommonDatautils.findDistrictById(dist._id.toString())
if(d){
def cityId = cityCodeDataMap.get(d.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
if(map){
row['M'] = map.avg_price
}
}
}
}
})
}
def addPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
if(row['I'] && !row['P']){
row['I'] = row['I'].toString().trim()
def d = CommonDatautils.findDistrictById(row['I'])
if(d){
row['K'] = d.city
row['L'] = d.region
row['M'] = d.name
row['T'] = d.estate_type2
row['U'] = d.cat
row['V'] = d.status
def cityMap = CommonDatautils.findThirdPartyPrice(d.city, d.region, d.name, 202303,202310)
def priceMap = cityMap.get(d.city+"##"+d.region+"##"+d.name)
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['N'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['N'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['O'] = priceMap.get("anjuke_esf_price")
}
}
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
if(dbo){
row['P'] = dbo.avg_price
}
def cityId = cityCodeDataMap.get(d.city)
def map = CcbCommonUtils.findAvgPrice(row['I'], cityId)
if(map){
row['Q'] = map.avg_price
}
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-10-01')
if(tradePrice){
row['R'] = tradePrice.toInteger()
}
def gxdPrice = CommonDatautils.findGXDPrice(d.city, d.region, d.name)
d['S'] = gxdPrice
}
}
})
}
def standard(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['G']){
row['G'] = row['F'].replace('·','').replace("小区","").replace("花园","").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
// println "[match] city:"+city+" region:"+region+" address:"+address
// def jsonObject = FangjiaClientUtils.newStandard(row['G'])
// if(jsonObject != null && jsonObject.precise!=-1) {
// row['Q'] = jsonObject.get("name")
// row['R'] = jsonObject.get("id")
// }
}
})
}
def addThirdPrice(path){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = CommonDatautils.findThirdPartyPrice(row['D'], row['E'], row['F'])
def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['L'] = priceMap.get("lianjia_esf_price")
}
if(priceMap.containsKey("beike_esf_price")){
row['M'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("anjuke_esf_price")){
row['N'] = priceMap.get("anjuke_esf_price")
}
}
})
}
}
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
//广州,佛山 重庆
static main(args) {
@@ -50,7 +50,7 @@ public class BeiKeCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver = new ChromeDriver()
def citys = getAllCityUrls(driver)
driver = getLogin(driver, "https://sh.ke.com/")
@@ -93,19 +93,28 @@ public class BeiKeCrawlByChrome {
}
def checkDocument(driver, url){
driver.get(url)
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
try{
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
println "异常Url:"+url
}
}
return Jsoup.parse(s);
return driver;
}
def crawlData(driver, citys){
@@ -130,7 +139,8 @@ public class BeiKeCrawlByChrome {
def newUrl = url+"/chengjiao/"
driver.get(newUrl)
Thread.sleep(10000)
def doc = checkDocument(driver, newUrl);
driver = checkDocument(driver, newUrl);
def doc = Jsoup.parse(driver.getPageSource())
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -142,8 +152,9 @@ public class BeiKeCrawlByChrome {
regionText = "昆山"
}
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println url +" 详情链接:"+detailUrls.size()
def map = getPageDeail(driver, newCity, href, regionText)
driver = map.driver
println url +" 详情链接:"+map.detailUrls.size()
}
}
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
@@ -191,9 +202,9 @@ public class BeiKeCrawlByChrome {
def detailUrls = []
while(!stop){
def url = cityUrl + "pg${page}/"
driver.get(url)
// def doc = Jsoup.parse(driver.getPageSource());
def doc = checkDocument(driver, url);
driver = checkDocument(driver, url);
def doc = Jsoup.parse(driver.getPageSource())
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
@@ -203,10 +214,11 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
println "开始查库 "+new Date().toString()
def dbo = findTrade(durl , "贝壳找房")
println "结束查库"+new Date().toString()
if (!dbo) {
parse(city, e, region, driver, null)
driver = parse(city, e, region, driver, null)
}
// else if(dbo && !dbo.lift){
// parse(city, e, region, driver, dbo._id)
@@ -225,7 +237,10 @@ public class BeiKeCrawlByChrome {
stop = true;
}
}
return detailUrls
def m = [:]
m.driver = driver
m.detailUrls = detailUrls
return m
}
def parse(city,e, region, driver, id){
@@ -233,8 +248,10 @@ public class BeiKeCrawlByChrome {
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
driver = checkDocument(driver, detailUrl);
def detailDoc = Jsoup.parse(driver.getPageSource())
println "打开获取页面 "+new Date().toString()
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -335,18 +352,21 @@ public class BeiKeCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println "解析页面完成 "+new Date().toString()
println detail
if(id){
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
}else{
DEAL_PRICE_COLL.save(detail)
}
println "保存修改完成 "+new Date().toString()
}
}
}catch(Exception ex){
ex.printStackTrace()
println e.select(".title a").attr("href")
}
return driver
}
@@ -469,16 +489,16 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
// if (citys.contains(e.text())){
if (citys.contains(e.text())){
list << [e.text(), href]
// }
}
}
}
return list
}
def getLogin(driver, url){
// WebDriver driver = new ChromeDriver()
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
@@ -494,17 +514,25 @@ public class BeiKeCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("17148854005");
elemPassword.sendKeys("12qwaszx");
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
// elemUsername.sendKeys("16283166371");
// elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(10000)
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
return driver
}
@@ -34,6 +34,7 @@ public class LianjiaCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -42,6 +43,7 @@ public class LianjiaCrawlByChrome {
// task.addDealRegion();
// task.standardRegion()
// task.updadteDealPriceField()
}
def updadteDealPriceField(){
@@ -64,7 +66,7 @@ public class LianjiaCrawlByChrome {
}
def run1(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
@@ -122,7 +124,7 @@ public class LianjiaCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
// cityMap = cityMap.reverse()
driver = getLogin(driver, "https://sh.lianjia.com/")
@@ -206,17 +208,11 @@ public class LianjiaCrawlByChrome {
}
def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/"
try{
driver.get(newUrl)
}catch(Exception e){
println newUrl +" 链接有问题!"
continue
}
// driver.get(newUrl)
//def detailUrls = getPageUrls(driver, cityUrl)
Thread.sleep(10000)
def doc = checkDocument(driver, newUrl);
// Thread.sleep(10000)
driver = checkDocument(driver, newUrl);
def doc = Jsoup.parse(driver.getPageSource())
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -226,8 +222,9 @@ public class LianjiaCrawlByChrome {
regionText = "昆山"
}
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
def map = getPageDeail(driver, newCity, href, regionText)
driver = map.driver
println newUrl +" 详情链接:"+map.detailUrls.size()
}
}
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
@@ -267,30 +264,42 @@ public class LianjiaCrawlByChrome {
}
def checkDocument(driver, url){
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
try{
driver.get(url)
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
return Jsoup.parse(s);
return driver;
}
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "pg${page}/"
driver.get(url)
// driver.get(url)
//def detailUrls = getPageUrls(driver, cityUrl)
def doc = checkDocument(driver, url);
driver = checkDocument(driver, url);
def doc = Jsoup.parse(driver.getPageSource())
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
@@ -303,7 +312,7 @@ public class LianjiaCrawlByChrome {
detailUrls << durl
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver, null)
driver = parse(city, e, regionText, driver, null)
}
// else if(dbo && !dbo.lift){
// parse(city, e, regionText, driver, dbo._id)
@@ -326,7 +335,10 @@ public class LianjiaCrawlByChrome {
stop = true;
}
}
return detailUrls
def m = [:]
m.driver = driver
m.detailUrls = detailUrls
return m
}
def parse(city,e, regionText, driver, id){
@@ -334,8 +346,11 @@ public class LianjiaCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if(price){
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
// driver.get(detailUrl)
driver = checkDocument(driver, detailUrl);
def detailDoc = Jsoup.parse(driver.getPageSource())
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -450,6 +465,7 @@ public class LianjiaCrawlByChrome {
ex.printStackTrace()
println e.select(".title a").attr("href")
}
return driver;
}
@@ -571,7 +587,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href]
}
}
@@ -593,16 +609,23 @@ public class LianjiaCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("17148854005");
elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
// elemUsername.sendKeys("17740818975");
// elemPassword.sendKeys("13qeadzc");
// elemUsername.sendKeys("16283166371");
// elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(15000)
Thread.sleep(20000)
}catch(Exception e){
Thread.sleep(10000)
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
return driver
}
@@ -3,7 +3,10 @@ package com.fangjia.script.gj.paimai
import com.alibaba.fastjson.JSON
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
@@ -35,7 +38,8 @@ class PaimaiTest {
// task.updateFieldPaimai()
// task.statCity()
// task.jsStat()
task.selectData()
// task.selectData()
task.randomExtractData()
}
def selectData(){
@@ -768,4 +772,38 @@ class PaimaiTest {
}
}
}
def randomExtractData(){
def runedcity = ["广州","重庆","南京","佛山","东莞","扬州","镇江","南通","惠州","珠海","连云港","淮安","清远","云浮","上海","深圳","苏州","徐州","盐城","宿迁","肇庆","韶关","汕尾","常州","湛江","泰州","无锡","江门","汕头","中山","合肥","河源","阳江","梅州","茂名","揭阳","福州","潮州"]
def citys = CommonDatautils.findAllCity()
def datas = []
datas << ["城市", "标地物", "面积", "URL", "结束时间", "成交价"]
for(def c : citys){
if(runedcity.contains(c)){
continue
}
def q = new BasicDBObject()
q.put("city", c)
q.put("status", "已成交")
q.put("estateType","住宅")
q.put("area", new BasicDBObject('$exists', true))
q.put("title",new BasicDBObject('$exists', true) )
def dbcursor = paimai43_COLL.find(q).sort(new BasicDBObject("finishDate",-1)).limit(100)
def rows = []
for(def d : dbcursor){
if(d.title && !d.title.contains("商") && !d.title.contains("铺") && !d.title.contains("车位")){
def date = DateUtils.formatDateByPattern(d.finishDate, "yyyy-MM-dd")
rows << [d.city, d.title, d.area, d.url, date, d.tradePrice]
}
if(rows.size()>=40){
break;
}
}
datas.addAll(rows)
println c +" "+rows.size()
}
ExcelUtils.write("D:\\数据\\北交所\\拍卖案例-住宅-测试.xlsx", datas, "数据1")
}
}
@@ -9,8 +9,19 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 价格对比,第三方,成交,建行价格,自已家价格
*/
class AddPriceTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -26,10 +37,120 @@ class AddPriceTest {
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
addFjwPrice(filePath)
// addFjwPrice(filePath)
exportPkPrice()
// addRegion()
}
def addRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
d['C'] = dist.region
})
}
def exportPkPrice(){
//"南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安",
def citys = ["宿迁"]
for(def c : citys){
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","alias"])
println c+" district:"+districts.size()
def fjwPriceMap =[:]// CommonDatautils.queryAllDistrictPrice(c, DateUtils.parseFromTightByPattern("2023-10-08","yyyy-MM-dd"))
def datas = addAllPrice(districts, fjwPriceMap)
ExcelUtils.write("D:\\数据\\上海农总行\\江苏省价格比对.xlsx", datas, c)
println c +" 完成"
}
}
def addAllPrice(districts, fjwPriceMap){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def rets = new CopyOnWriteArrayList()
rets << ["id", "城市","区域","名称","建行价格","计算方式","可信度","房价网ToB价格","计算方式","可信度","近6个月成交均价"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createPriceData(cityCodeDataMap, dist, fjwPriceMap)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createPriceData(cityCodeDataMap, d, fjwPriceMap){
def row = []
row << d._id.toString()
row << d.city
row << d.region
row << d.name
def cityId = cityCodeDataMap.get(d.city)
def map = findAvgPrice(d._id.toString(), cityId)
if(map){
row << map.avg_price
row << map.calc_type
row << map.credit_value
}else{
row << ""
row << ""
row << ""
}
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
// def dbo = fjwPriceMap.get(d.city+"##"+d.region+"##"+d.name)
if(dbo){
row << dbo.avg_price
if(dbo.value_type == 1){
row << "指导价"
}else if(dbo.value_type == 2){
row << "自身样本"
}else if(dbo.value_type == 3){
row << "第三方"
}else if(dbo.value_type == 4){
row << "交易数据"
}else if(dbo.value_type == 5){
row << "周边楼盘"
}else if(dbo.value_type == 6){
row << "推导"
}else if(dbo.value_type == 9){
row << "神经网络"
}else if(dbo.value_type == 10){
row << "无方式"
}else{
row << ""
}
row << dbo.credit
}else{
row << ""
row << ""
row << ""
}
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-09-01')
if(tradePrice){
row << tradePrice.toInteger()
}else{
row << ''
}
return row;
}
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -0,0 +1,80 @@
package com.fangjia.script.gj.test
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
//def json = GaoDeApiUtils.geo("上海市", "上海市浦东新区永泰路1138弄11号")
//def locationStr = null
//if(json && json.count>=1){
// def m = json.geocodes[0]
// if(m){
// locationStr = m.location
// }
//}
def location = ["lng":121.526016,"lat":31.143047]
//def rows = []
//rows << ["城市","区域","名称","地址","距离","均价"]
//if(location && location.lat){
//
// def dbos = CommonDatautils.getDistrictArround("上海", 121.526016, 31.143047)
// for(def d : dbos){
//
// def dbo = CommonDatautils.findDistrictPrice(d.city, d.region, d.name);
// def avg_price = null
// if(dbo){
// avg_price = dbo.avg_price
// }
//
// def distance = GeoUtils.calculateDistance(location.lng, location.lat, d.lng/10000000000.0d, d.lat/10000000000.0)
//
// rows << [d.city, d.region, d.name, d.address, distance.intValue(), avg_price, d.status]
// }
//
//// def map = CommonDatautils.findMaxLatLngMinLatLng(location.lat, location.lng, 1000);
//// def ret = CommonDatautils.findInRangeDist("上海", "district", (map.maxLat * 10000000000d).toLong(), (map.minLat * 10000000000d).toLong(), (map.maxLng * 10000000000d).toLong(), (map.minLng * 10000000000d).toLong());
////
//// if(ret){
//// for(def row : ret){
//// if(row.lat2 && row.lng2){
//// def lat2 = row.lat2/10000000000.0;
//// def lng2 = row.lng2/10000000000.0;
//// region = row.region;
//// def distance = GeoHelper.GetDistance(location.lat, location.lng, lat2, lng2)
//// if(distance <= 1000){
//// def dbo = CommonDatautils.findDistrictPrice(row.city, row.region, row.name);
//// def avg_price = null
//// if(dbo){
//// avg_price = dbo.avg_price
//// }
//// rows << [row.city, row.region, row.name, row.address, distance.intValue(), avg_price]
//// }
//// }
//// }
//// }
//
// ExcelUtils.write("D:\\数据\\临时\\上海市浦东新区永泰路1138弄11号.xlsx", rows, "一公里楼盘")
//}
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{d->
def loc = d['AB']
if(loc){
def arrs = loc.split(",")
def distance = GeoUtils.calculateDistance(location.lng, location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
d['AD'] = distance
}
})
//def distance = GeoUtils.calculateDistance(121.526016, 31.143047, 111.202375, 35.592230)
//println distance
@@ -0,0 +1,180 @@
package com.fangjia.script.gj.township
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class StreetProcess {
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
final DBCollection POLYLINE235 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new StreetProcess();
// println RegionData.formatRegion("鞍山","铁东区")
task.run()
}
def run(){
//
// def region = RegionData.formatRegion("三明市", "三元区")
// println region
// standard()
// matchStreet()
synchTownship()
}
def synchTownship(){
def q = new BasicDBObject()
// q.put("city_name","福州市")
// q.put("shape",new BasicDBObject('$exists', true))
def dbcursor = TOWNSHIP.find(q)
for(def d : dbcursor){
def udata = new BasicDBObject()
// udata.put("shape", d.shape)
udata.put("area_station", d.area_station)
TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
def matchStreet(){
def polylineMap = findStreetPolyline()
def q = new BasicDBObject()
// q.put("fangjia_city", "福州")
// q.put("shape",new BasicDBObject('$exists', false))
def dbcursor = TOWNSHIP.find(q)
def index = 0
for(def d : dbcursor){
index++
def streetMap = polylineMap.get(d.fangjia_city+"##"+d.fangjia_area)
if(streetMap){
def alias = d.alias
if(!alias){
alias = []
}
if(!alias.contains(d.street_name)){
alias << d.street_name
}
for(def streetName : alias){
def polyline = streetMap.get(streetName)
if(polyline){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
def udata = new BasicDBObject()
udata.put("shape", resultMap)
TOWNSHIP.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
println index + " "+ d.get("_id")
break;
}
}
}
}
}
def findStreetPolyline(){
def q = new BasicDBObject()
// q.put("fjw_city","福州")
def dbcursor = POLYLINE235.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.fjw_city+"##"+d.fjw_region
def streetMap = map.get(key)
if(!streetMap){
streetMap = [:]
map.put(key, streetMap)
}
streetMap.put(d.town, d.polyline)
}
return map;
}
def standard(){
def q = new BasicDBObject()
q.put("fjw_city",new BasicDBObject('$exists', false))
def dbcursor = POLYLINE235.find(q)
def index = 0
for(def d : dbcursor){
index ++
def city = RegionData.formatCity(d.city)
def region = RegionData.formatRegion(d.city, d.region)
// def street = null
// if(city && region){
// street = RegionData.formatStreet(city, region, d.town)
// }
def udata = new BasicDBObject()
udata.put("fjw_city", city)
udata.put("fjw_region", region)
// if(street){
// udata.put("fjw_street", street)
// }
POLYLINE235.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println index
}
}
def standard1(){
def list = TOWNSHIP.find(new BasicDBObject()).toList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def city = RegionData.formatCity(d.city_name)
if(city){
def fangjia_province = ProvinceData.getProvince(city)
def region = RegionData.formatRegion(city, d.area_name)
def udata = new BasicDBObject()
udata.put("fangjia_city", city)
udata.put("fangjia_area", region)
udata.put("fangjia_province", fangjia_province)
TOWNSHIP.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println totalCount.get()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
}
}
@@ -0,0 +1,86 @@
package com.fangjia.script.gj.village
import java.util.Arrays;
class BaiduCoordinateConverter {
static double[] array1 = [75, 60, 45, 30, 15, 0]
static double[][] array2 = [
[-0.0015702102444, 111320.7020616939, 1704480524535203.0, -10338987376042340.0, 26112667856603880.0, -35149669176653700.0, 26595700718403920.0, -10725012454188240.0, 1800819912950474.0, 82.5],
[0.0008277824516172526, 111320.7020463578, 647795574.6671607, -4082003173.641316, 10774905663.51142, -15171875531.51559, 12053065338.62167, -5124939663.577472, 913311935.9512032, 67.5],
[0.00337398766765, 111320.7020202162, 4481351.045890365, -23393751.19931662, 79682215.47186455, -115964993.2797253, 97236711.15602145, -43661946.33752821, 8477230.501135234, 52.5],
[0.00220636496208, 111320.7020209128, 51751.86112841131, 3796837.749470245, 992013.7397791013, -1221952.21711287, 1340652.697009075, -620943.6990984312, 144416.9293806241, 37.5],
[-0.0003441963504368392, 111320.7020576856, 278.2353980772752, 2485758.690035394, 6070.750963243378, 54821.18345352118, 9540.606633304236, -2710.55326746645, 1405.483844121726, 22.5],
[-0.0003218135878613132, 111320.7020701615, 0.00369383431289, 823725.6402795718, 0.46104986909093, 2351.343141331292, 1.58060784298199, 8.77738589078284, 0.37238884252424, 7.45]
]
static double[] array3 = [12890594.86, 8362377.87, 5591021, 3481989.83, 1678043.12, 0];
static double[][] array4 = [
[1.410526172116255e-8, 0.00000898305509648872, -1.9939833816331, 200.9824383106796, -187.2403703815547, 91.6087516669843, -23.38765649603339, 2.57121317296198, -0.03801003308653, 17337981.2],
[-7.435856389565537e-9, 0.000008983055097726239, -0.78625201886289, 96.32687599759846, -1.85204757529826, -59.36935905485877, 47.40033549296737, -16.50741931063887, 2.28786674699375, 10260144.86],
[-3.030883460898826e-8, 0.00000898305509983578, 0.30071316287616, 59.74293618442277, 7.357984074871, -25.38371002664745, 13.45380521110908, -3.29883767235584, 0.32710905363475, 6856817.37],
[-1.981981304930552e-8, 0.000008983055099779535, 0.03278182852591, 40.31678527705744, 0.65659298677277, -4.44255534477492, 0.85341911805263, 0.12923347998204, -0.04625736007561, 4482777.06],
[3.09191371068437e-9, 0.000008983055096812155, 0.00006995724062, 23.10934304144901, -0.00023663490511, -0.6321817810242, -0.00663494467273, 0.03430082397953, -0.00466043876332, 2555164.4],
[2.890871144776878e-9, 0.000008983055095805407, -3.068298e-8, 7.47137025468032, -0.00000353937994, -0.02145144861037, -0.00001234426596, 0.00010322952773, -0.00000323890364, 826088.5]
];
public static double[] convertor(double x, double y, double[] param) {
double t = param[0] + param[1] * Math.abs(x);
double cc = Math.abs(y) / param[9];
double cf = param[2] + param[3] * cc + param[4] * cc * cc + param[5] * cc * cc * cc + param[6] * cc * cc * cc * cc + param[7] * cc * cc * cc * cc * cc + param[8] * cc * cc * cc * cc * cc * cc;
t *= (x < 0) ? -1 : 1;
cf *= (y < 0) ? -1 : 1;
return [t, cf];
}
public static double[] bdLatlng2Mercator(double lat, double lng) {
double[] arr = null;
double nLat = (lat > 74) ? 74 : lat;
nLat = (nLat < -74) ? -74 : nLat;
for (int i = 0; i < array1.length; i++) {
if (lat >= array1[i]) {
arr = array2[i];
break;
}
}
if (arr == null) {
for (int i = array1.length - 1; i >= 0; i--) {
if (lat <= -array1[i]) {
arr = array2[i];
break;
}
}
}
if (arr == null) {
throw new IllegalArgumentException("Invalid latitude");
}
double[] res = convertor(lng, lat, arr);
return [res[0], res[1]]
}
public static double[] bdMercator2Latlng(double x, double y) {
double[] arr = null;
double npX = Math.abs(x);
double npY = Math.abs(y);
for (int i = 0; i < array3.length; i++) {
if (npY >= array3[i]) {
arr = array4[i];
break;
}
}
if (arr == null) {
throw new IllegalArgumentException("Invalid y value");
}
double[] res = convertor(npX, npY, arr);
return [res[0], res[1]]
}
public static void main(String[] args) {
// Example usage
double latitude = 4832281.76
double longitude = 12959065.3
// double[] mercatorPoint = bdLatlng2Mercator(latitude, longitude);
// System.out.println("Mercator Point: " + Arrays.toString(mercatorPoint));
double[] latlngPoint = bdMercator2Latlng(longitude, latitude);
System.out.println("Latlng Point: " + Arrays.toString(latlngPoint));
}
}
@@ -0,0 +1,380 @@
package com.fangjia.script.gj.village
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
def filePath = "D:\\数据\\南京\\南京银行测试-新跑.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
def communityObj = null
if(row['E']){
communityObj = findCommunityTown(community_CONN, row['C'], row['D'], row['E'])
}
if(communityObj){
row['M'] = communityObj.province_code
row['N'] = communityObj.city_code
row['O'] = communityObj.area_code
row['P'] = communityObj.street_code
}
if(row['D']){
if(row['D'].toString().endsWith("区")){
row['F'] = "非农"
}else{
if(communityObj){
if(communityObj.has_gov && communityObj.has_gov=="是"){
row['I'] = "是"
row['F'] = "非农"
}else{
row['I'] = "否"
if(row['E']){
def townObj = isTown(row['C'], row['D'], row['E'])
if(townObj){
row['J'] = "是"
row['F'] = "非农"
}else{
row['F'] = "农"
}
}
}
}
}
}
});
//def filePath = "D:\\数据\\光大\\全量.xlsx"
//ExcelUtils.modify(filePath, { row ->
// if(!row['O']){
// def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A'])
// if(jsonObj && jsonObj.result){
// row['B'] = jsonObj.result.countyLevelCity
// if(row['B']){
// if(row['B']=="是"){
// row['C'] = "否"
// }else if(row['B']=="否"){
// row['C'] = "是"
// }
// }
// row['D'] = jsonObj.result.district
// row['E'] = jsonObj.result.districtCode
// if(jsonObj.result.administrativeTowns && jsonObj.result.administrativeTowns=="是"){
// row['F'] = jsonObj.result.administrativeTowns
// row['G'] = jsonObj.result.town
// row['H'] = jsonObj.result.townCode
// }
//
// if(jsonObj.result.countyGov && jsonObj.result.countyGov=="是"){
// row['I'] = jsonObj.result.countyGov
// row['J'] = jsonObj.result.town
// row['K'] = jsonObj.result.townCode
// }
//
// if(jsonObj.result.devDistrict && jsonObj.result.devDistrict=="是"){
// row['L'] = jsonObj.result.devDistrict
// row['M'] = jsonObj.result.devDistrictName
// }
// row['N'] = jsonObj.result.addrVerify
// row['O'] = jsonObj.result.province
// row['P'] = jsonObj.result.city
// row['Q'] = jsonObj.result.committee
// row['R'] = jsonObj.result.typeName
// row['S'] = jsonObj.result.splitCommittee
// row['T'] = jsonObj.result.splitStreet
// row['U'] = jsonObj.result.splitStreetNum
// row['V'] = jsonObj.result.poiName
// row['W'] = jsonObj.result.dev
// }
// }
// def retMap = SignatureUtils.getAliAddressStrc(row['A'])
// println row['A'] +" " + retMap.toString()
// if(retMap){
// row['Y'] = retMap.prov
// row['Z'] = retMap.city
// row['AA'] = retMap.district
// row['AB'] = retMap.town
// row['AC'] = retMap.devzone
// row['AD'] = retMap.road
// row['AE'] = retMap.roadno
// row['AF'] = retMap.poi
// row['AG'] = retMap.subpoi
// }
// })
//ExcelUtils.multiThreadingModify(filePath, { row ->
//
// if(row['AA'] && !row['AO']){
// if(row['Y']){
// if(row['Y'].endsWith("区")){
// row['AE'] = "否"
// row['AF'] = "是"
// }else {
// row['AE'] = "是"
// row['AF'] = "否"
// }
// }
//
// if(row['AA']){
// def dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], row['AA'])
// if(!dev_name && row['AA'].contains("经济开发区")){
// dev_name = row['AA'].replace("经济开发区","经济技术开发区")
// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
// }
// if(!dev_name && row['AA'].contains("经济技术开发区")){
// dev_name = row['AA'].replace("经济技术开发区","经济开发区")
// dev_name = isDev(development_zone_CONN, row['W'], row['X'], row['Y'], dev_name)
// }
// if(dev_name){
// row['AO'] = "是"
// row['AP'] = dev_name
// }
// }
//
// if(row['W'] && row['X'] && row['Y'] && row['Z']){
// def townObj = isTown(row['X'], row['Y'], row['Z'])
// if(townObj){
// row['AI'] = "是"
// row['AJ'] = row['Z']
// }
//
// def communityObj = findCommunityTown(community_CONN, row['X'], row['Y'], row['Z'])
// if(communityObj){
// row['AL'] = isGov(communityObj)
// if(row['AL'] && row['AL'] =="是"){
// row['AM'] = row['Z']
// row['AN'] = communityObj.street_code
// }
// row['AG'] = communityObj.area_name
// row['AH'] = communityObj.area_code
//
// if(row['AI'] && row['AI'] == "是"){
// row['AK'] = communityObj.street_code
// }
// }else{
// def towns = findAllTowns(community_CONN, row['X'], row['Y'])
// def retObj = findBestSimilar(row['Z'], towns)
// if(retObj && retObj.score>=0.5){
// row['AQ'] = retObj.name
// }
// }
// }
// }
//
//})
//ExcelUtils.multiThreadingModify(filePath, { row ->
// if(row['F'] && !row['T']){
// def q = new BasicDBObject()
// q.put("province", row['B'])
// def rets = development_zone_CONN.find(q).toList()
// def maxScore = 0;
// def map = [:]
// if(rets){
// for(def r : rets){
// def score1 = CosineSimilarAlgorithm.getSimilarityRatio(row['B']+row['C']+row['F'], r.name);
// if(score1>maxScore){
// map.name = r.name
// map.score = score1
// maxScore = score1
// }
// }
// if(map){
// row['W'] = map.name
// row['X'] = map.score
// }
// }
// }
//})
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['C'] && row['R']){
def obj = findFirstData(community_CONN, row['C'], row['D'], row['R'])
if(obj){
row['S'] = obj.street_code
}
}
})
def findBestSimilar(town, towns){
def map = [:];
def maxScore = 0
for(def name : towns){
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(town, name);
if(score1>maxScore){
map.name = name
map.score = score1
maxScore = score1
}
}
return map
}
def isDev(development_zone_CONN, def prov, def city, def region, def dev){
def q = new BasicDBObject()
q.put("province", prov)
dev = dev.replace("(","").replace(")","").replace("市","")
if(dev.contains(region) && region.toString().endsWith("区")){
def newRegion = region.replace("区","")
dev = dev.replace(region, newRegion)
}
Pattern pattern = Pattern.compile('^.*' + dev + '.*$', Pattern.CASE_INSENSITIVE);
q.put("name",new BasicDBObject('$regex', pattern))
def rets = development_zone_CONN.find(q).toList()
def developmentZone = null
if(rets){
def devList = []
devList.add(dev);
try{
devList.add(region.replace("县","").replace("区","").replace("市","") + dev);
}catch(e){
println region +" "+ e.getMessage()
}
try{
devList.add(city.replace("市","") + region.replace("县","").replace("区","").replace("市","") + dev);
}catch(e){
println city+" "+region+" "+e.getMessage()
}
try{
devList.add(prov.replace("省","") + city.replace("市","")+ region.replace("县","").replace("区","").replace("市","") + dev);
}catch(e){
println prov+" "+city+" "+region+" "+e.getMessage()
}
try{
devList.add(city.replace("市","") + dev);
}catch(e){
println e
}
try{
devList.add(prov.replace("省","")+ city.replace("市","") + dev);
}catch(e){
println e
}
try{
devList.add(prov.replace("省","")+ region.replace("县","").replace("区","").replace("市","")+ dev);
}catch(e){
println e
}
devList.add(prov.replace("省","")+ dev);
for(def zone : rets){
String name = zone.name.replace("省","").replace("市","").replace("县","")
if(!name.endsWith("区")){
name = name.replace("区","");
}
if(devList.contains(name)){
developmentZone = zone;
break;
}
}
}
if(developmentZone){
return developmentZone.name;
}
return null;
}
def isTown( city, region, town){
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
// obj = it
// },"192.168.0.89")
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
def body = """
{
"query": {
"bool":{
"must":[
{"term": {"city.keyword":"${city}"}},
{"term": {"district.keyword":"${region}"}},
{"term": {"town.keyword":"${town}"}}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
def obj = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
obj = hits[0]
}
}
println(response)
return obj
}
def findCommunityTown(community_CONN, city, region, town){
def q = new BasicDBObject()
q.put("city_name", city)
q.put("area_name", region)
Pattern pattern = Pattern.compile(town + '.*$', Pattern.CASE_INSENSITIVE);
q.put("street_name",new BasicDBObject('$regex', pattern))
def rets = community_CONN.find(q).toList()
def obj = null
if(rets){
obj = rets[0]
}
return obj;
}
def findAllTowns(community_CONN, city, region){
def q = new BasicDBObject()
q.put("city_name", city)
q.put("area_name", region)
def rets = community_CONN.find(q).toList()
def towns = []
for(def t : rets){
if(!towns.contains(t.street_name)){
towns << t.street_name
}
}
return towns;
}
def findFirstData(community_CONN, city, region, town){
def q = new BasicDBObject()
q.put("city_name", city)
q.put("area_name", region)
q.put("street_name", town)
def obj = community_CONN.findOne(q)
return obj
}
def isGov(obj){
String hasGovFlag = null;
if(obj){
hasGovFlag = obj.has_gov;
}
if(StringUtils.isNotBlank(hasGovFlag) && hasGovFlag.equals("是")){
return "是"
}else {
return "否"
}
}
@@ -0,0 +1,53 @@
package com.fangjia.script.gj.village.ailiyun
import com.aliyun.tea.*;
class AnliyunApiUtils {
/**
* 使用AK&SK初始化账号Client
* @param accessKeyId
* @param accessKeySecret
* @return Client
* @throws Exception
*/
public static com.aliyun.iot20180120.Client createClient(String accessKeyId, String accessKeySecret) throws Exception {
com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config()
// 必填,您的 AccessKey ID
.setAccessKeyId(accessKeyId)
// 必填,您的 AccessKey Secret
.setAccessKeySecret(accessKeySecret);
// Endpoint 请参考 https://api.aliyun.com/product/Iot
config.endpoint = "iot.cn-beijing.aliyuncs.com";
return new com.aliyun.iot20180120.Client(config);
}
public static void main(String[] args_) throws Exception {
java.util.List<String> args = java.util.Arrays.asList(args_);
// 请确保代码运行环境设置了环境变量 ALIBABA_CLOUD_ACCESS_KEY_ID 和 ALIBABA_CLOUD_ACCESS_KEY_SECRET。
// 工程代码泄露可能会导致 AccessKey 泄露,并威胁账号下所有资源的安全性。以下代码示例使用环境变量获取 AccessKey 的方式进行调用,仅供参考,建议使用更安全的 STS 方式,更多鉴权访问方式请参见:https://help.aliyun.com/document_detail/378657.html
com.aliyun.iot20180120.Client client = createClient(Config.accessKey, Config.accessKeySecret);
com.aliyun.iot20180120.models.CreateDeviceGroupRequest createDeviceGroupRequest = new com.aliyun.iot20180120.models.CreateDeviceGroupRequest()
.setIotInstanceId("your_value")
.setSuperGroupId("your_value")
.setGroupDesc("your_value")
.setGroupName("your_value");
try {
// 复制代码运行请自行打印 API 的返回值
client.createDeviceGroupWithOptions(createDeviceGroupRequest, new com.aliyun.teautil.models.RuntimeOptions());
} catch (TeaException error) {
// 错误 message
System.out.println(error.getMessage());
// 诊断地址
System.out.println(error.getData().get("Recommend"));
com.aliyun.teautil.Common.assertAsString(error.message);
} catch (Exception _error) {
TeaException error = new TeaException(_error.getMessage(), _error);
// 错误 message
System.out.println(error.getMessage());
// 诊断地址
System.out.println(error.getData().get("Recommend"));
com.aliyun.teautil.Common.assertAsString(error.message);
}
}
}
@@ -0,0 +1,14 @@
package com.fangjia.script.gj.village.ailiyun
/* 服务端API签名配置文件
*
* @author: ali
* @version: 0.1 2018-08-08 08:23:54
*/
public class Config {
// AccessKey信息
public static String accessKey = "LTAI5t7UKjXaoGxzet2Vh3bB";
public static String accessKeySecret = "2avjPxmI7FlIRHvMC5C1UuAnd9iG8z";
public final static String CHARSET_UTF8 = "utf8";
}
@@ -0,0 +1,203 @@
package com.fangjia.script.gj.village.ailiyun
import com.aliyuncs.CommonRequest
import com.aliyuncs.CommonResponse
import com.aliyuncs.DefaultAcsClient
import com.aliyuncs.IAcsClient
import com.aliyuncs.exceptions.ClientException
import com.aliyuncs.http.FormatType
import com.aliyuncs.http.MethodType
import com.aliyuncs.profile.DefaultProfile
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import net.sf.json.JSONObject
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.net.URLEncoder
import java.rmi.ServerException;
import java.util.Map;
import java.util.TreeMap;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
/**
* 服务端API签名
*
* @author: ali
* @version: 0.1 2018-06-21 20:47:05
*/
public class SignatureUtils {
private final static String CHARSET_UTF8 = "utf8";
private final static String ALGORITHM = "HmacSHA1";
private final static String SEPARATOR = "&";
public static Map<String, String> splitQueryString(String url)
throws URISyntaxException, UnsupportedEncodingException {
URI uri = new URI(url);
String query = uri.getQuery();
final String[] pairs = query.split("&");
TreeMap<String, String> queryMap = new TreeMap<String, String>();
for (String pair : pairs) {
final int idx = pair.indexOf("=");
final String key = idx > 0 ? pair.substring(0, idx) : pair;
if (!queryMap.containsKey(key)) {
queryMap.put(key, URLDecoder.decode(pair.substring(idx + 1), CHARSET_UTF8));
}
}
return queryMap;
}
public static String generate(String method, Map<String, String> parameter, String accessKeySecret)
throws Exception {
String signString = generateSignString(method, parameter);
System.out.println("signString---" + signString);
byte[] signBytes = hmacSHA1Signature(accessKeySecret + "&", signString);
String signature = newStringByBase64(signBytes);
System.out.println("signature----" + signature);
if ("POST".equals(method))
return signature;
return URLEncoder.encode(signature, "UTF-8");
}
public static String generateSignString(String httpMethod, Map<String, String> parameter) throws IOException {
TreeMap<String, String> sortParameter = new TreeMap<String, String>();
sortParameter.putAll(parameter);
String canonicalizedQueryString = UrlUtil.generateQueryString(sortParameter, true);
if (null == httpMethod) {
throw new RuntimeException("httpMethod can not be empty");
}
StringBuilder stringToSign = new StringBuilder();
stringToSign.append(httpMethod).append(SEPARATOR);
stringToSign.append(percentEncode("/")).append(SEPARATOR);
stringToSign.append(percentEncode(canonicalizedQueryString));
return stringToSign.toString();
}
public static String percentEncode(String value) {
try {
return value == null ? null
: URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
"~");
} catch (Exception e) {
}
return "";
}
public static byte[] hmacSHA1Signature(String secret, String baseString) throws Exception {
if (StringUtils.isEmpty(secret)) {
throw new IOException("secret can not be empty");
}
if (StringUtils.isEmpty(baseString)) {
return null;
}
Mac mac = Mac.getInstance("HmacSHA1");
SecretKeySpec keySpec = new SecretKeySpec(secret.getBytes(CHARSET_UTF8), ALGORITHM);
mac.init(keySpec);
return mac.doFinal(baseString.getBytes(CHARSET_UTF8));
}
public static String newStringByBase64(byte[] bytes) throws UnsupportedEncodingException {
if (bytes == null || bytes.length == 0) {
return null;
}
return new String(Base64.encodeBase64(bytes, false), CHARSET_UTF8);
}
// public static void main(String[] a){
// Map<String, String> map = new HashMap<String, String>();
// Calendar calendar = Calendar.getInstance()
// def startTime = DateUtil.formatByPattern(new Date(), "yyyy-MM-dd'T'HH:mm:ss.SSSZ")
// // 公共参数
// map.put("Format", "JSON");
// map.put("Version", "2019-11-18");
// map.put("AccessKeyId", Config.accessKey);
// map.put("SignatureMethod", "HMAC-SHA1");
// map.put("Timestamp", startTime);
// map.put("SignatureVersion", "1.0");
// map.put("SignatureNonce", System.currentTimeMillis().toString());
//// map.put("RegionId", "cn-shanghai");
// // 请求参数
// map.put("Action", "StreetStd");
// map.put("ServiceCode", "addrp");
// map.put("AppKey", "Xczmmbxlur2g");
// map.put("Text","山东省烟台市莱阳市龙门东路1号0027、0028")
// try {
// String signature = SignatureUtils.generate("GET", map, Config.accessKeySecret);
// if(signature){
// map.put("signature", signature);
// }
// String content = new HttpRequest().get("https://address-purification.cn-hangzhou.aliyuncs.com", map, "utf-8")
// JSONObject result = JSONObject.fromObject(content)
// System.out.println("最终signature: " + signature);
// } catch (Exception e) {
// e.printStackTrace();
// }
// System.out.println();
// }
public static def getAliAddressStrc(def address) {
DefaultProfile defaultProfile = DefaultProfile.getProfile(
"cn-hangzhou",
Config.accessKey,
Config.accessKeySecret);
IAcsClient client = new DefaultAcsClient(defaultProfile);
// 创建API请求并设置参数
CommonRequest request = new CommonRequest();
request.setSysDomain("address-purification.cn-hangzhou.aliyuncs.com");
request.setSysVersion("2019-11-18");
request.setSysProduct("address-purification");
request.setSysLocationProduct("addrp");
//设置要请求的服务
request.setSysAction("StreetStd");
//设置请求方式
request.setSysMethod(MethodType.POST);
request.setHttpContentType(FormatType.FORM);
//请求头
request.putHeadParameter("x-acs-signature-method", "HMAC-SHA1");
request.putHeadParameter("x-acs-signature-nonce", UUID.randomUUID().toString());
//设置项目token
request.putBodyParameter("AppKey", "Xczmmbxlur2g");
//设置请求参数
request.putBodyParameter("Text", address);
// request.putBodyParameter("DefaultCity", "杭州市");
request.putQueryParameter("AcceptLanguage", "zh-CN");
try {
CommonResponse response = client.getCommonResponse(request);
def data = response.getData();
def result = JSONObject.fromObject(data)
if(result && result.Data){
def data1 = JSONObject.fromObject(result.Data)
def std = data1.street_std
if(std){
def arrs = std.split("\t")
def map = [:]
for(def s : arrs){
def paramValues = s.split("=")
map.put(paramValues[0], paramValues[1])
}
return map
}
}
} catch (ServerException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ClientException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
}
@@ -0,0 +1,55 @@
package com.fangjia.script.gj.village.ailiyun
import java.net.URLEncoder;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
/**
* URL处理类
*
* @author: ali
* @version: 0.1 2018-06-21 20:40:52
*/
public class UrlUtil {
private final static String CHARSET_UTF8 = "utf8";
public static String urlEncode(String url) {
if (!StringUtils.isEmpty(url)) {
try {
url = URLEncoder.encode(url, "UTF-8");
} catch (Exception e) {
System.out.println("Url encode error:" + e.getMessage());
}
}
return url;
}
public static String generateQueryString(Map<String, String> params, boolean isEncodeKV) {
StringBuilder canonicalizedQueryString = new StringBuilder();
for (Map.Entry<String, String> entry : params.entrySet()) {
if (isEncodeKV)
canonicalizedQueryString.append(percentEncode(entry.getKey())).append("=")
.append(percentEncode(entry.getValue())).append("&");
else
canonicalizedQueryString.append(entry.getKey()).append("=").append(entry.getValue()).append("&");
}
if (canonicalizedQueryString.length() > 1) {
canonicalizedQueryString.setLength(canonicalizedQueryString.length() - 1);
}
return canonicalizedQueryString.toString();
}
public static String percentEncode(String value) {
try {
// 使用URLEncoder.encode编码后,将"+","*","%7E"做替换即满足API规定的编码规范。
return value == null ? null
: URLEncoder.encode(value, CHARSET_UTF8).replace("+", "%20").replace("*", "%2A").replace("%7E",
"~");
} catch (Exception e) {
}
return "";
}
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,149 @@
package com.fangjia.script.gj.village
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
//def s = getTownshipInfo("118.702988,39.740979")
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
//ExcelUtils.multiThreadingModify(filePath, { row ->
//// if(!row['I']){
//// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
//// if(rets){
//// def obj = findOneByName(rets, row['C']+"人民政府")
//// if(obj){
//// row['I'] = obj.lng
//// row['J'] = obj.lat
//// println row['C'] + " "+obj
//// }
//// }
//// }
//
//// if(row['D'] && !row['F']){
//// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
//// def map = ["lon":row['D'], "lat":row['E']]
//// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
//// row['F'] = township
//// }
//
//// if(row['F'] && !row['J']){
//// def q = new BasicDBObject()
//// q.put("city_name", row['B'])
//// q.put("area_name", row['C'])
//// def streetName = row['F']
//// q.put("street_name", streetName)
//// def count = COMMUNITY_CONN.count(q);
//// row['J'] = count
////// row['H'] = streetName
//// }
//// if(!row['H']){
//// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
//// if(gaodeMap && gaodeMap.township){
//// row['H'] = gaodeMap.township
//// }
//// }
//
//// if(row['J'] && row['J'].toString().toDouble()>0){
//// def up_query = new BasicDBObject()
//// up_query.put("city_name", row['B'])
//// up_query.put("area_name", row['C'])
//// up_query.put("street_name", row['F'])
//// def udata = new BasicDBObject()
//// udata.put("has_gov", "是")
//// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
//// }
//
//
//})
def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
for(def d : objs){
if(!d.name_pinyin){
if(d.name){
println d._id
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
def udata = new BasicDBObject()
udata.put("name_pinyin", py1)
if(d.location){
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
udata.put("location", ["lat":lat, "lon":lon])
}
def q = new BasicDBObject()
q.put("_id", d._id)
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
}
}
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.district = district
map.township = township
return map;
}
def findOneByName(def rets, name){
def obj = null
for(def r : rets){
if(r.name == name){
obj = r
break;
}
}
return obj
}
def findTownRange(city, TOWN_POLYLINECOLL){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
@@ -0,0 +1,47 @@
package com.fangjia.script.gj.village
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community_new_2023", MONGO100_URL)
filePath = "D:\\Êý¾Ý\\ɿũ\\18Íò×îÖÕ-id.xlsx"
def index = 0
ExcelUtils.multiThreadingModify(filePath, { d->
if(d['A']){
def q = new BasicDBObject("_id", d['A'])
def dbo = COMMUNITY_CONN.findOne(q)
if(dbo){
index++
println index
COMMUNITY_CONN.remove(q)
}else{
// def udata = new BasicDBObject()
// udata.put("_id" , d['A'])
// udata.put("province" , d['A'])
// udata.put("province_code" , d['A'])
// udata.put("city_code" , d['A'])
// udata.put("area_name" , d['A'])
// udata.put("area_code" , d['A'])
// udata.put("street_name" , d['A'])
// udata.put("street_code" , d['A'])
// udata.put("name" , d['A'])
// udata.put("code" , d['A'])
// udata.put("type_code" , d['A'])
// udata.put("type_name" , d['A'])
// udata.put("location_type" , d['A'])
// udata.put("geohash" , d['A'])
// udata.put("location" , d['A'])
// udata.put("new_poi_name" , d['A'])
// udata.put("new_area_name" , d['A'])
// udata.put("city_name" , d['A'])
// COMMUNITY_CONN.save(udata);
}
}
})
@@ -0,0 +1,562 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 厦门银行
* 福建省,重庆市poi数据处理
*/
@BatchOwner(Watcher.GaoJuan)
class GaoDePoiDataProcess extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static def version = new Date().getTime()
public static def TAGS = ["加油站","其它能源站","加气站","洗车场","充电站","餐饮相关场所","休闲餐饮场所","购物相关场所","商场","超级市场","家电电子卖场","花鸟鱼虫市场","家居建材市场","综合市场","特色商业街","生活服务场所","邮局","事务所","人才市场","洗浴推拿场所","体育休闲服务场所","运动场馆","娱乐场所","度假疗养场所","休闲场所","医疗保健服务场所","综合医院","专科医院","诊所","急救中心","疾病预防机构","住宿服务相关","宾馆酒店","旅馆招待所","风景名胜相关","公园广场","风景名胜","产业园区","政府及社会团体相关","政府机关","外国机构","社会团体","公检法机构","交通车辆管理","工商税务机构","科教文化场所","博物馆","展览馆","会展中心","美术馆","图书馆","科技馆","天文馆","文化宫","档案馆","文艺团体","传媒机构","学校","科研机构","培训机构","驾校","机场相关","银行","银行相关","保险公司","证券公司","财务公司","公司企业","知名企业","公司","工厂","农林牧渔基地","收费站","服务区","标志性建筑物","地名地址信息","热点地名","紧急避难场所"]
public final static DBCollection COMPANY_206_COLL = MongoUtils.getCollectionByName("business", "company_huzhou", "192.168.0.206",27020)
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
def fullCityMap = null
static main(args) {
runBatch(new GaoDePoiDataProcess(), args)
}
@Override
void process(Object args) {
// println ProvinceData.getProvince("成都")
// synchMapPoi();
synchCompany()
// poiCatProcess()
// addressAndCatProcess()
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
}
def sysnchPoi(){
def q = new BasicDBObject()
q.put("version", new BasicDBObject('$gte',1701273600000))
def dbcursor = MAPPOI_CONN_TEST.find(q).batchSize(20000)
def ret = []
def i = 0
for(def d : dbcursor){
i++
ret << d
if(ret.size()%50000==0){
println "第 "+(i/50000)+" 个"+ret.size()
MAPPOI_CONN.insert(ret);
ret = [];
}
}
if(ret.size()>0){
MAPPOI_CONN.insert(ret);
}
}
def synchCompany(){
fullCityMap = CommonDatautils.queryFullNameCity()
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
def citys = ["杭州","宁波","温州","嘉兴","台州","绍兴","金华","舟山","衢州","丽水"]
for(def sampleCity : citys){
def cityObj = fullCityMap.get(sampleCity);
def city = null
if(cityObj){
city = cityObj.city
}
def townMap = findTownRange(city)
def companyDatas = findCompanyDatas(city)
println city+ " 加载公司数据:"+companyDatas.size()
def rows = running(city, companyDatas, "company", townMap)
save(city, rows)
// saveToMG(city, rows)
println "完成"
}
}
def findCompanyDatas(city){
def q = new BasicDBObject()
q.put("done", 1)
q.put("geo_city", city)
// q.put("geo_level", new BasicDBObject('$in', ['门牌号','兴趣点']))
return COMPANY_206_COLL.find(q).toList()
}
def synchMapPoi(){
def runedCity = []
fullCityMap = CommonDatautils.queryFullNameCity()
def citys = CommonDatautils.findAllCity().reverse();
for(def sampleCity : citys){
def cityObj = fullCityMap.get(sampleCity);
def city = null
if(cityObj){
city = cityObj.city
}
if(city && !runedCity.contains(city)){
def townMap = findTownRange(city)
//// def results = CommonDatautils.findAmapDetailBySql(city)
def results = CommonDatautils.findNewAmapDetailBySql(city)
println "city: "+city +" town: "+townMap.size()+" poi:"+results.size()
def rows = running(city, results, "map", townMap)
//// save(city, rows)
saveToMG(city, rows)
}
}
}
def running(city,results, type, townMap){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def prov = ProvinceData.getProvince(city.replace("市",""))
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(type == "map"){
rets.addAll(createData(prov, map, townMap))
}else if(type == "company"){
rets.addAll(createCompanyData(map, townMap))
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createCompanyData(map, townMap){
def retMap = [:]
retMap.data_id = map._id.toString()
retMap.fjw_province = map.province_name
retMap.fjw_city = map.city_name
try{
def fjw_region = RegionData.formatRegion(map.city_name, map.geo_district)
if(fjw_region){
retMap.fjw_district = fjw_region
}
}catch(Exception e){
println map.geo_district
}
if(!retMap.fjw_district ){
retMap.fjw_district = RegionData.formatRegion(map.city_name, map.region_name)
}
retMap.province = map.geo_province
retMap.city = map.geo_city
retMap.district = map.geo_district
retMap.cat = "企业"
retMap.name = map.ent_name
// def pointList = townMap.get(retMap.fjw_district)
def locationStr = map.geo_location
if(locationStr){
def arrs = locationStr.split(",")
retMap.township = InsidePolygon.isInArea2(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), townMap)
def gd_location = [:]
gd_location.lon = DataHandleUtils.getDoubleRound(6, arrs[0].toString().toDouble())
gd_location.lat = DataHandleUtils.getDoubleRound(6, arrs[1].toString().toDouble())
retMap.location = gd_location
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
retMap.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
retMap.gps_location = gpslocation
}
if(!retMap.township){
def gaodeMap = getTownshipInfo(locationStr)
if(gaodeMap && gaodeMap.township){
retMap.township = gaodeMap.township
}
if(!retMap.fjw_district){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district)
}
}
if(!fjw_region && !retMap.fjw_district){
println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap
}
retMap.fjw_district = fjw_region
}
}
retMap.address = map.reg_addr
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.ent_name, " ")
retMap.version = version
retMap.status = 1
retMap.credit_code = map.unified_code
return retMap
}
def createData(province, map, townMap){
def fjw_province = province
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
map.fjw_province = fjw_province
map.fjw_city = fjw_city
if(map.cat){
def arrs = map.cat.split(";|:")
map.type = arrs[0]
if(arrs.size()>=2){
map.secondary_type = arrs[1]
}
if(arrs.size()>=3){
map.small_type = arrs[2]
}
if(arrs.size()>3){
map.cat = arrs[0]+";"+arrs[1]+";"+arrs[2]
}
}
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
map.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
map.gps_location = gpslocation
}
if(!map.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
if(!fjw_region){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
}
}
if(!fjw_region){
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
}
map.fjw_district = fjw_region
}
map.fjw_district = fjw_region
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
map.version = version
if(map.secondary_type && TAGS.contains(map.secondary_type)){
map.status = 0
}else{
map.status = 1
}
return map
}
def save(city,rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "[${city}] 结束时间 "+new Date()
}
def saveToMG(city, rows){
def rets = []
for(def i=0;i<rows.size();i++){
def m = rows[i]
def obj = new BasicDBObject()
m.each{k,v->
obj.put(k,v)
}
rets << obj
if(i%50000==0){
MAPPOI_CONN.insert(rets)
rets.clear()
}
}
if(rets.size()>0){
MAPPOI_CONN.insert(rets)
rets.clear()
}
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.district = district
map.township = township
return map;
}
def findTownRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
// def regionMap = townMap.get(d.fjw_region)
// if(!regionMap){
// regionMap = [:]
// townMap.put(d.fjw_region, regionMap)
// }
// def townInfoMap = regionMap.get(d.town)
// if(townInfoMap){
// townInfoMap = [:]
// regionMap.put()
// }
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
def poiCatProcess(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\数据.xlsx",{d->
def cat = d['B']
if(cat && cat.size()>50){
println cat
def arrs = cat.split(";")
def newCat = ""
if(arrs.size()>3){
newCat = arrs[0]+";"+arrs[1]+";"+arrs[2]
def udata = new BasicDBObject()
udata.put("cat",newCat)
udata.put("type",arrs[0])
udata.put("secondary_type",arrs[1])
udata.put("small_type",arrs[2])
MAPPOI_CONN.update(new BasicDBObject("cat", cat), new BasicDBObject('$set', udata), false, true)
}
}
})
}
def addressAndCatProcess(){
def datas = FileUtils.readTxt("C:\\Users\\gaojuan\\Downloads\\BIGDATA_POI_v1.bad")
datas.each{d->
def arrs = d.split("@\\|@")
if(arrs){
def udata = new BasicDBObject()
// def address = arrs[5]
// if(address.size()>=120){
// udata.put("address", "")
// }
def cat = arrs[8]
def catArrs = cat.split(";")
if(catArrs.size()>3){
cat = catArrs[0]+";"+catArrs[1]+";"+catArrs[2]
if(cat.size()>=45){
println "取前3依然超:"+cat
}else{
udata.put("type",catArrs[0])
udata.put("secondary_type",catArrs[1])
udata.put("small_type",catArrs[2])
udata.put("cat", cat)
}
}else{
println cat
}
if(udata){
MAPPOI_CONN.update(new BasicDBObject("_id", arrs[0]), new BasicDBObject('$set', udata), false, true)
}
}
}
}
def synchNetwork(){
def datas = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx")
def rets = []
for(def d : datas){
def location = d['I']
def locMap = [:]
if(location){
def arrs = location.split(",")
locMap.lon = arrs[0].toString().toDouble()
locMap.lat = arrs[1].toString().toDouble()
}
rets << ["id":new ObjectId().toString(), "name":d['B'], "address":d['C'], "location":locMap]
}
saveEs("xm_network", rets)
}
def saveEs(index,rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient("192.168.0.12")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest(index, "_doc", map.id).source(map, XContentType.JSON));
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
}
@@ -0,0 +1,162 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeoUtils
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class NetworkDistanceBatch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "192.168.0.43")
static main(args) {
runBatch(new NetworkDistanceBatch(), args)
}
@Override
void process(Object args) {
def networks = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx", 0, 1)
def citys = ["厦门","三明","宁德","泉州","福州","漳州","南平","龙岩","莆田","重庆"]
for(def c : citys){
run(c, "district", networks)
}
// saveMapData()
}
def saveMapData(){
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\厦门银行鹭通支行.xlsx",{d->
if(d['C']){
def arrs = d['C'].split(",")
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
def obj = new BasicDBObject()
obj.put("network_name", d['B'])
obj.put("ntk_gd_lng", arrs[0].toString().toDouble())
obj.put("ntk_gd_lat", arrs[1].toString().toDouble())
obj.put("ntk_bd_lng", lng2)
obj.put("ntk_bd_lat", lat2)
obj.put("comm_id", d['D'])
obj.put("city", d['E'])
obj.put("region", d['F'])
obj.put("name", d['G'])
obj.put("distance", d['I'].toString().toDouble().toInteger())
obj.put("time", d['K'].toString().toDouble().toInteger())
obj.put("gh_distance", d['J'].toString().toDouble().toInteger())
def dist_loc_arr = d['H'].split(",")
def baidu_dist = BaiDuApiUtils.geoconv(dist_loc_arr[0].toString().toDouble(), dist_loc_arr[1].toString().toDouble(), 3)
obj.put("dist_gd_lng", dist_loc_arr[0].toString().toDouble())
obj.put("dist_gd_lat", dist_loc_arr[1].toString().toDouble())
obj.put("dist_bd_lng", baidu_dist.x)
obj.put("dist_bd_lat", baidu_dist.y)
MAP_NETWORK_COLL.save(obj)
}
})
}
def run(city, cat, networks){
def datas = findDistrictData(city, cat, ["city","region", "name", "category", "location"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def dist = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = createData(dist, networks)
if(map){
rets << map
}
// println city +" "+cat +" "+totalCount.get() +"/"+datas.size()+" "+dist.name+" "+dist._id
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
saveDatas(rets);
println cat +" "+city +" 总量:"+datas.size()+" 在5公里内总量:"+rets.size()
}
def saveDatas(def rets){
NETWORK_COLL.insert(rets)
}
def createData(dist, networks){
def lastRest = null
def minDistance = Integer.MAX_VALUE
for(def row : networks){
def locationstr = row['E']
if(locationstr){
def arrs = locationstr.split(",")
def distance = GeoUtils.calculateDistance(dist.location.lon, dist.location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
if(distance <= minDistance){
minDistance = distance
lastRest = row
}
}
}
if(minDistance<=5000){
def obj = new BasicDBObject()
obj.put("_id", dist._id);
obj.put("city", dist.city);
obj.put("region", dist.region);
obj.put("name", dist.name);
obj.put("distance", (int)minDistance);
obj.put("network", lastRest['B'])
return obj
}
return null;
}
def findDistrictData(def city, cat, fields){
DBObject q = new BasicDBObject();
q.put("status", 0);
q.put("category", cat);
q.put("city", city);
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
return SEAWEED_CONN.find(q, keys).toList();
}
}
@@ -0,0 +1,35 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
@BatchOwner(Watcher.GaoJuan)
class PoiAndSeaweedExportCSV extends BaseBatch{
static main(args) {
runBatch(new PoiAndSeaweedExportCSV(), args);
}
@Override
public void process(Object args) {
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
poiExport(citys)
}
def poiExport(citys){
def fileName = "/home/goojia/xiamen/BIGDATA_POI.csv"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\厦门\\BIGDATA_POI.csv"
}
for(def c : citys){
def rets = CommonDatautils.findNewAmapDetail(c)
FileUtils.batchCreateFileBySplitChar(rets, fileName, null, "utf-8", true, "@|@");
}
}
}
@@ -0,0 +1,692 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class PoiWash {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
// DBCollection MAPPOI_CONN2 = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL)
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static def townsMap = [:]
public static def regions = []
public static def nameKeys = null
static main(args) {
def task = new PoiWash()
// task.updateId()
// task.run()
// task.synch()
// task.check()
task.deduplicate()
// task.updateEsMapPoi()
// task.checkSpecialSymbols()
// task.splitRoad()
}
def updateId(){
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
for(def city : citys){
def datas = findPois(city)
update(datas)
}
}
def update(datas){
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
def count = datas.size()
for(def i=0;i<datas.size();i++){
def map = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("id", map._id.toString())
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+map.city+" 更改id"
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def synch(){
def rets = []
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
for(def city : citys){
def datas = findAllFieldPois(city)
println city +" "+datas.size()+" 同步"
for(def d : datas){
rets << d
if(rets.size()%10000==0){
MAPPOI_CONN2.insert(rets)
rets.clear()
}
}
}
if(rets.size()>0){
MAPPOI_CONN2.insert(rets)
rets.clear()
}
}
def checkSpecialSymbols(){
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
def citys = ["重庆市"]
for(def city : citys){
findAllFieldPois(city)
println city +" 完成"
}
}
def specialSymbols(city, results){
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def columns = ["province","city","district","name","address","cat","business_area","fjw_province","fjw_city","type","secondary_type","small_type","township", "fjw_district","spell_name"]
def udata = new BasicDBObject()
map.each{k,v->
if(columns.contains(k)){
if(v instanceof Double){
udata.put(k, "")
}
}
}
if(udata){
updateTotalCount.addAndGet(1)
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def run(){
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"
//"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市"
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
for(def city : citys){
townsMap = findAllTownship(city)
regions = findAllRegions(city)
def datas = findPois(city)
println city +' poi: '+datas.size()+" towns:"+ townsMap.get(RegionData.formatCity(city)).size()
process(city, datas)
// updateOldRegion(city)
println city +" 完成"
}
}
def process(city, results){
// def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def newAddr = washing(map)
if(map.address!=newAddr){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("address", newAddr)
MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
}
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
// return rets;
}
def washing(data){
if(data.address){
def addr = data.address.replace('[]', '').replace(' ','').replace(data.province,'')
if(data.province){
addr = addr.replace(data.province,'')
}
if(data.city){
addr = addr.replace(data.city,'')
}
if(data.district){
if(!addr.contains(data.district+"路")){
addr = addr.replace(data.district,'')
}
}
if(data.township){
try{
addr = addr.replace(data.township,'')
}catch(Exception e){
println data.id+" "+data.township
}
}
if(addr && addr.startsWith('-')){
addr = addr.replace('---','').replace('--','').replace('-','')
}
if(addr && addr.isNumber()){
return null;
}
if(addr){
for(def r : regions){
addr = addr.replace(r, "")
}
}
if(addr){
def towns = townsMap.get(data.fjw_city)
for(def t : towns){
if(addr.contains(t)){
addr = addr.replace(t, "")
continue
}
}
}
return addr
}
return null;
}
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("test", 1)
// q.put("name", '(三星)企业设立登记第二受理点')
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("address", 1)
keys.put("fjw_city", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
rets << d
if(rets.size()>=300000){
print "."
break;
}
}
return rets;
}
def findAllFieldPois(def city){
def q = new BasicDBObject()
q.put("city", city)
def dbcursor = MAPPOI_CONN.find(q).batchSize(10000)
def rets = []
for(def d : dbcursor){
rets << d
if(rets.size()>0 && rets.size()%100000==0){
print "."
specialSymbols(city, rets)
rets.clear()
}
}
if(rets.size()>0){
specialSymbols(city, rets)
rets.clear()
}
return rets;
}
def updateOldRegion(def city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("fjw_city", 1)
keys.put("fjw_district", 1)
keys.put("township", 1)
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000)
def rets = []
def distMap = [:]
for(def d : dbcursor){
// def region = distMap.get(d.fjw_district)
// if(!region){
// distMap.put(d.fjw_district, d.district)
// }
//
// if(d.district instanceof Double){
// def newRegion = distMap.get(d.fjw_district)
// if(newRegion){
// println d.id +" "+d.fjw_district+" "+newRegion
// def udata = new BasicDBObject()
// udata.put("district", newRegion)
// MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
// }
// }
if(d.township instanceof Double){
def udata = new BasicDBObject()
udata.put("township", "")
MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
println d.id
}
}
return rets;
}
def findAllRegions(cityName){
def q = new BasicDBObject()
if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){
q.put("province_name", cityName)
}else{
q.put("city_name", cityName)
}
def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000)
def set = new HashSet()
for(def d : dbcursor){
set << d.area_name
}
return set
}
def findAllTownship(cityName){
def q = new BasicDBObject()
if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){
q.put("province_name", cityName)
}else{
q.put("city_name", cityName)
}
def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000)
def map = [:]
for(def d : dbcursor){
def city = d.fangjia_city
if(!city || city instanceof Double){
city = RegionData.formatCity(d.city_name)
}
def towns = map.get(city)
if(!towns){
towns = []
map.put(city, towns)
}
def alias = d.alias
towns << d.street_name
if(alias){
towns.addAll(alias)
}
// towns.unique()
}
return map
}
def deduplicate(){
//"三明市","漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
def total = 0
def upindex = 0
for(def city : citys){
nameKeys = new HashSet()
def q = new BasicDBObject()
q.put("city", city)
q.put("status", new BasicDBObject('$ne', 2))
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
total++
def key = d.city +"##"+d.district+"##"+d.name+"##"+d.location.lon+"##"+d.location.lat
if(!nameKeys.contains(key)){
nameKeys << key
}else{
upindex++
def udata = new BasicDBObject()
udata.put("status", 2)
MAPPOI_CONN.update(new BasicDBObject('id',d.id),new BasicDBObject('$set', udata))
println "[${upindex}/${total}] Delete : "+d.city +" "+ d.district+" "+d.name
}
}
}
}
def updateEsMapPoi(){
// "三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市",
def citys = ["龙岩市","莆田市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
for(def city : citys) {
def datas = findPoiByQuery(city)
for(def d : datas){
updateMapPoi(d.id)
println city +" "+d.id
}
}
}
def splitRoad(){
//"三明市",
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
for(def city : citys) {
def datas = findPoiByQuery(city, ['city','address'])
splitRoadRun(datas)
println city +" 完成"
}
}
def splitRoadRun(results){
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
if(jsonObject && jsonObject.containsKey("result")){
def dataMap = jsonObject.get("result")
if(dataMap.get("street")){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("street", dataMap.get("street"))
if(dataMap.get("streetNumber")){
udata.put("street_number", dataMap.get("streetNumber"))
}
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
}
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def updateMapPoi(def id){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/map_poi/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.status=2",
"lang": "painless"
},
"query": {
"term": {
"id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}
def findPoiByQuery(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("status", 2)
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
def rets = []
for (def d : dbcursor) {
rets << d
}
return rets;
}
def findPoiByQuery(city, fields){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
fields.each{
keys.put(it, 1)
}
def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(5000)
def rets = []
for (def d : dbcursor) {
rets << d
}
return rets;
}
def check(){
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
for(def city : citys){
def stop = false
while(!stop){
def datas = findPois(city)
if(datas.size()>0){
processUpdate(city, datas)
}else{
stop = true
}
}
println city +" 完成"
}
}
def processUpdate(city, results){
// def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
def updateTotalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def count = results.size()
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def udata = new BasicDBObject()
def cat = map.cat
if(cat && cat.contains(":")){
def arrs = map.cat.split(":")
udata.put("type",arrs[0])
if(arrs.size()>=2){
udata.put("secondary_type",arrs[1])
}
if(arrs.size()>=3){
udata.put("small_type",arrs[2])
}
if(arrs.size()>3){
udata.put("cat",arrs[0]+";"+arrs[1]+";"+arrs[2])
}
}
if(map.bd_location){
def new_bd_lon = DataHandleUtils.getDoubleRound(6, map.bd_location.lon)
def new_bd_lat = DataHandleUtils.getDoubleRound(6, map.bd_location.lat)
def new_bd_map = ["lon":new_bd_lon, "lat":new_bd_lat]
udata.put("bd_location", new_bd_map)
}
if(map.gps_location){
def new_gps_lon = DataHandleUtils.getDoubleRound(6, map.gps_location.lon)
def new_gps_lat = DataHandleUtils.getDoubleRound(6, map.gps_location.lat)
def new_gps_map = ["lon":new_gps_lon, "lat":new_gps_lat]
udata.put("gps_location", new_gps_map)
}
if(map.location){
def new_lon = DataHandleUtils.getDoubleRound(6, map.location.lon)
def new_lat = DataHandleUtils.getDoubleRound(6, map.location.lat)
def new_map = ["lon":new_lon, "lat":new_lat]
udata.put("location", new_map)
}
if(map.address && (map.address.contains("(") || map.address.contains("("))){
def s1 = null
def addr = map.address.replace("(", "(").replace(")", ")")
if(addr.contains("(") && addr.contains(")")){
s1 = addr.substring(addr.indexOf("("),addr.indexOf(")")+1 )
}
if(s1){
def new_addr = addr.replace(s1, "")
if(DataHandleUtils.isStandardRoad(new_addr)){
udata.put("address", new_addr)
}else{
if(DataHandleUtils.isStandardRoad(s1)){
udata.put("address", s1.replace("(", "").replace(")", ""))
}
}
}
}
if(udata){
updateTotalCount.addAndGet(1)
// udata.put("test", 2)
MAPPOI_CONN.update(new BasicDBObject('id',map.id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
}
// if(map.address){
// def newAddr = DataUtil.toDBCcase(map.address)
//
// if(newAddr.startsWith(";")){
// newAddr = newAddr.substring(1)
// }
// if(newAddr.contains("-") && newAddr.replace("-","").isNumber()){
// newAddr = ""
// }
//
// if(map.address!=newAddr){
// updateTotalCount.addAndGet(1)
// def udata = new BasicDBObject()
// udata.put("address", newAddr)
// MAPPOI_CONN.update(new BasicDBObject('_id',map.id),new BasicDBObject('$set', udata))
// println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
// }
// }
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
latch.await(3, TimeUnit.MINUTES);
// return rets;
}
}
@@ -0,0 +1,87 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class PostCodeSynchBatch extends BaseBatch{
public static final DBCollection post_code_coll = MongoUtils.getCollectionByName("crawl", "post_code", "192.168.0.206", 27020)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection POST_CODE_CONN = MongoUtils.getCollectionByName2("fangjia", "postcode", MONGO100_URL)
def fullCityMap = null
def mcaDistMap = null
static main(args) {
runBatch(new PostCodeSynchBatch(), args);
}
@Override
public void process(Object args) {
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
def datas = findAllDatas();
for(def d : datas){
def map = [:]
map._id = d._id
def area_code = null
map.province = d.province
def short_city = RegionData.formatCity(d.city)
if(short_city){
def data = fullCityMap.get(short_city)
if(data){
map.city = data.city
map.city_code = data.mca_city_code
map.province_code = data.mca_province_code
}
def region = d.district.replace(d.province, "").replace(d.city, "")
def short_district = RegionData.formatRegion(short_city, region)
def distData = mcaDistMap.get(short_city+"#"+short_district)
if(distData){
map.district = distData.district
map.district_code = distData.mca_district_code
}else{
println d.city +" "+d.district+" 民政部中没有找到相应的区域全称"
}
}else{
println d.city +" 没有标准化出来"
}
map.city_zip_code = d.city_code
map.district_zip_code = d.district_code
map.address = d.addr
map.addr_zip_code = d.addr_code
updateData(map)
}
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
if(k!="_id"){
udata.put(k,v)
}
}
POST_CODE_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
}
def findAllDatas(){
def q = new BasicDBObject()
def dbcursor = post_code_coll.find(q)
return dbcursor.toList();
}
}
@@ -0,0 +1,369 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import com.fangjia.script.common.DataHandleUtils
class SeaweedDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
public static def provinceMap = CommonDatautils.findAllCityProvince()
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
public static def priceMap = null
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
def fullCityMap = null
def mcaDistMap = null
static main(args) {
runBatch(new SeaweedDataSynch(), args);
}
@Override
public void process(Object args) {
run()
// println "全国district, shop, office同步完成"
//民政部编码上传
//codeRun()
//println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate()
// addTest()
}
def addTest(){
def q = new BasicDBObject()
def ret = []
def dbcurosr = community_100_CONN.find(q).batchSize(20000)
for(def d : dbcurosr){
def m = new BasicDBObject()
m.put("province" , d.province_name)
m.put("province_code", d.province_code)
if(['北京市','上海市','重庆市','天津市'].contains(d.province_name)){
m.put("city" , d.province_name)
}else if(d.area_name=="自治区直辖县级行政区划" || d.area_name=="省直辖县级行政区划"){
m.put("city" , d.area_name)
}else{
m.put("city" , d.city_name)
}
m.put("city_code" ,d.city_code)
m.put("district" , d.area_name)
m.put("district_code" , d.area_code)
m.put("town" , d.street_name)
m.put("town_code" , d.street_code)
m.put("committee" , d.name)
m.put("committee_code" , d.code)
m.put("location",d.location)
m.put("type_code" ,d.type)
if(d.type=="111"){
m.put("type" , "主城区")
}else if(d.type=="112"){
m.put("type" , "城乡结合区")
}else if(d.type=="121"){
m.put("type" , "镇中心区")
}else if(d.type=="122"){
m.put("type" , "镇乡结合区")
}else if(d.type=="123"){
m.put("type" , "特殊区域")
}else if(d.type=="210"){
m.put("type" , "乡中心区")
}else if(d.type=="220"){
m.put("type" , "村庄")
}
ret << m
if(ret.size()%50000==0){
stat_administrative_division_100_CONN.insert(ret)
ret = []
}
}
if(ret.size()>0){
stat_administrative_division_100_CONN.insert(ret)
}
}
def run(){
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
// priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity().reverse()
def index = 0
for(def c : citys){
index++
updating(c, "district")
updating(c, "office")
updating(c, "shop")
println index +" "+ c + " 完成"
}
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = createData(dist)
//往100 上同步
updateData(map)
//往12 es上同步
// def newUrl = targetUrl +"_doc/"+map.id
// CommonDatautils.httpSaveData(newUrl, map)
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
if(k!="_id"){
udata.put(k,v)
}
}
SEAWEED_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
}
def createData(dist){
def map = [:]
map._id = dist._id
def area_code = null
def province = provinceMap.get(dist.city)
if(province){
map.province = ProvinceData.getFullProvince(province)
}
map.fangjia_province = province
def data = fullCityMap.get(dist.city)
if(data){
map.city = data.city
area_code = data.mca_city_code
}
map.fangjia_city = dist.city
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
if(distData){
map.region = distData.district
area_code = distData.mca_district_code
}else{
// map.region = dist.region
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
}
map.area_code = area_code
map.fangjia_region = dist.region
map.name = dist.name
map.address = dist.address
map.block = dist.block
if(dist.mapping_alias){
def aliasNames = []
for(def key : dist.mapping_alias){
if(key && !key.toString().equals("null")){
aliasNames << key
if(aliasNames.size()>5){
map.alias = aliasNames
break;
}else{
if(aliasNames){
map.alias = aliasNames
}
}
}
}
}
if(dist.addr_alias){
def names = []
for(def key : dist.addr_alias){
if(key && !key.toString().equals("null")){
names << key
if(names.size()>5){
map.addr_alias = names
break;
}else{
if(names){
map.addr_alias = names
}
}
}
}
}
map.category = dist.cat
map.estate_type = dist.estate_type2
map.status = dist.status
map.township = dist.township
map.finish_build_time = dist.create_year?dist.create_year.toString():null
if(dist.lng){
def loc = [:]
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
map.location = loc
}
if(map.location){
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
map.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
map.gps_location = gpslocation
}
}
map.build_type = dist.build_type
map.households = dist.household_count
map.committee = dist.committee
if(dist.estate_charge && dist.estate_charge.toString().isNumber()){
def fee = dist.estate_charge.toString().toFloat()
if(fee < 100){
map.property_fee = DataHandleUtils.getDoubleRound(2, fee)
}
}
if(dist.car_site_count && dist.car_site_count.toString().isNumber()){
map.car_site_count = dist.car_site_count.toString().toInteger()
}
if(dist.property_life && dist.property_life.toString().isNumber()){
map.property_life = dist.property_life.toString().toInteger()
}
map.house_property = dist.house_property
if(dist.build_area && dist.build_area.toString().isNumber()){
map.build_area = dist.build_area.toString().toDouble().toInteger()
}
if(dist.all_area && dist.all_area.toString().isNumber()){
map.floor_area = dist.all_area.toString().toDouble().toInteger()
}
map.build_structure = dist.build_structure
map.developer = dist.developer
map.property_company = dist.estate_company
map.plot_ratio = dist.cubage_percent
map.landscaping_ratio = dist.virescence_percent
if(dist.loudong_count && dist.loudong_count.toString().isNumber()){
map.building_count = dist.loudong_count.toString().toInteger()
}
map.schools = dist.schools
map.main_house_type = dist.main_type
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat())
}
if(priceMap){
def dataMap = priceMap.get(dist._id.toString())
if(dataMap){
map.avg_price = dataMap.avg_price
}
}
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
return map
}
def findAllDistrictSellPrice(){
def q = new BasicDBObject()
q.put("date", lastMonthDate)
def priceMap = [:]
DISTRICT_SELL_STAT.find(q).each{
priceMap.put(it.commid, it)
}
return priceMap
}
def codeRun(){
def q = new BasicDBObject()
def dbcursor = mca_administrative_division_coll.find(q)
def rets = []
for(def it : dbcursor){
def udata = new BasicDBObject()
udata.put("_id", it._id)
udata.put("province", it.province)
udata.put("city", it.city)
udata.put("district", it.district)
udata.put("mca_province_code", it.province_code)
udata.put("mca_city_code", it.city_code)
udata.put("mca_district_code", it.district_code)
udata.put("level", it.level)
udata.put("c_date", it.c_date)
rets << udata
// administrative_division_100_CONN.save(udata)
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
}
administrative_division_100_CONN.insert(rets)
}
}
@@ -0,0 +1,461 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.lixue.DB.City
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.*
import java.util.concurrent.atomic.AtomicLong
class SuggestionDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL)
public static def provinceMap = CommonDatautils.findAllCityProvince()
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
public static def priceMap = null
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def keys = new CopyOnWriteArraySet();
def fullCityMap = null
def mcaDistMap = null
static main(args) {
runBatch(new SuggestionDataSynch(), args);
}
@Override
public void process(Object args) {
run()
// updateCode()
// getExitsCity()
}
def getExitsCity(){
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
def cities = SUGGESTION_CONN.distinct("city", new BasicDBObject("type","poi"))
def simpleCities = []
for (def c :cities) {
def city = RegionData.formatCity(c)
simpleCities.add(city)
}
// print(simpleCities)
return simpleCities
}
def run() {
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"
def citys = CommonDatautils.findAllCity()
def index = 0
fullCityMap = queryFullNameCity()
mcaDistMap = queryDistNameCity()
def existCities = getExitsCity()
for (def c : citys) {
def allRets = []
// if(["上海","北京","广州","深圳","成都","杭州","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳","怀化","郴州","益阳","永州","娄底","湘西","长沙","常德","株洲","岳阳","衡阳","湘潭","张家界","邵阳","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁"].contains(c)){
if(existCities.contains(c)){
continue;
}
index++
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
def ret1 = updating(c, "district")
def ret2 = updating(c, "office")
def ret3 = updating(c, "shop")
// }
// if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
def ret4 = poiData(c)
// }
allRets.addAll(ret1)
allRets.addAll(ret2)
allRets.addAll(ret3)
allRets.addAll(ret4)
saveToMG(c, allRets)
println index + " " + c + " 完成 "+ allRets.size()
}
}
def saveToMG(city, rows){
def data = fullCityMap.get(city)
def q = new BasicDBObject("city", data.city)
def count = 0
try{
count = SUGGESTION_CONN.count(q)
}catch(e){
SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion_new", MONGO100_URL)
count = SUGGESTION_CONN.count(q)
}
// if(count>0){
// SUGGESTION_CONN.remove(q)
// }
def rets = []
for(def i=0;i<rows.size();i++){
def m = rows[i]
def obj = new BasicDBObject()
m.each{k,v->
obj.put(k,v)
}
rets << obj
if(i%50000==0){
SUGGESTION_CONN.insert(rets)
rets.clear()
}
}
if(rets.size()>0){
SUGGESTION_CONN.insert(rets)
rets.clear()
}
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
def key = dist.city+"#"+dist.region+"#"+dist.name
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!keys.contains(key)){
keys << key
def map = createData(dist)
//往100 上同步
// updateData(map)
rets << map
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
}
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(dist){
def map = [:]
map.id = dist._id.toString()
def province = provinceMap.get(dist.city)
if(province){
map.province = ProvinceData.getFullProvince(province)
}
def data = fullCityMap.get(dist.city)
map.city = data.city
def distData = mcaDistMap.get(dist.city+"#"+dist.region)
if(distData){
map.region = distData.district
}else{
map.region = dist.region
println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称"
}
map.name = dist.name
map.address = dist.address
if(dist.alias && dist.alias.size()>0){
map.alias = dist.alias
}
if(dist.addr_alias && dist.addr_alias.size()>0){
map.addr_alias = dist.addr_alias
}
map.type = dist.cat
if(dist.lng){
def loc = [:]
loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0)
map.location = loc
}
map.source = "FJW"
map.township = dist.township
return map
}
def findAllDistrictSellPrice(){
def q = new BasicDBObject()
q.put("date", lastMonthDate)
def priceMap = [:]
DISTRICT_SELL_STAT.find(q).each{
priceMap.put(it.commid, it)
}
return priceMap
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
if(k!="id"){
udata.put(k,v)
}
}
SUGGESTION_CONN.update(new BasicDBObject('_id', map.id), new BasicDBObject('$set', udata), true, false)
}
def poiData(city){
def townMap = CommonDatautils.findTownRange(city)
def data = fullCityMap.get(city)
if(data){
def results = CommonDatautils.findAmapDetailBySql(data.city)
def rets = createPoi(data.city, results, data, townMap)
return rets;
}
return []
}
def createPoi(fullCity, results, data, townMap){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
def townshipMap = findMapPoiTownshipMap(fullCity)
final CountDownLatch latch = new CountDownLatch(results.size())
for(def i=0;i<results.size();i++){
final def dist = results[i]
def key = dist.city+"#"+dist.district+"#"+dist.name
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!keys.contains(key)){
keys << key
def map = [:]
map.id = dist.id
map.province = dist.province
map.city = dist.city
map.region = dist.district
map.name = dist.name
map.address = dist.address
map.type = "poi"
map.cat = dist.cat
map.location = dist.location
map.source = "FJW"
if(townshipMap && townshipMap.containsKey(dist.id)){
def township = townshipMap.get(dist.id)
if(township){
map.township = township
}
}
if(!map.township){
map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap)
if(!map.township){
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
}
}
println map.township
//往100 上同步
// updateData(map)
rets << map
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
}
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def findMapPoiTownshipMap(fullCity){
def q = new BasicDBObject()
q.put("city", fullCity)
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("township", 1)
def dbcursor = MAPPOI_CONN.find(q, keys)
def map = [:]
for(def d : dbcursor){
map.put(d.id, d.township)
}
return map;
}
def findTownshipByBD(id){
def q = new BasicDBObject()
q.put("id", id)
def obj = MAPPOI_CONN.findOne(q)
return obj
}
def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
def fjwcity = null
if(d.level == "城市"){
fjwcity = RegionData.formatCity(d.city)
if(!fjwcity){
if(d.city == "市辖区"){
fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
d.city = d.province
}
}
}
}else if(d.level == "区域"){
if(d.administer == "省直辖县级行政区划"){
fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
d.city = d.district
}
}
}
if(fjwcity){
retMap.put(fjwcity, d)
}else{
println d.toString() +" 格式化不成功!"
}
}
return retMap;
}
def queryDistNameCity(){
def q = new BasicDBObject()
q.put("level","区域")
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
if(d.city == "省直辖县级行政区划"){
continue
}else if(d.city == "市辖区"){
def fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}else{
def fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}
}
return retMap;
}
def updateCode(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
for(def d : dbcursor){
def code = null
if(d.mca_district_code){
code = d.mca_district_code
}else if(d.mca_city_code){
code = d.mca_city_code
}else if(d.mca_province_code){
code = d.mca_province_code
}
DBObject udata = new BasicDBObject()
udata.put("code", code);
administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata));
}
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.district = district
map.township = township
return map;
}
}
@@ -45,10 +45,10 @@ class PdsUseStatisticBatch extends BaseBatch{
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [
"duweiliang@fangjia.com",
"yaojingmeng@fangjia.com",
"xuqi@fangjia.com",
"wangpeili@fangjia.com"
// "duweiliang@fangjia.com",
// "yaojingmeng@fangjia.com",
// "xuqi@fangjia.com",
"gaojuan@fangjia.com"
]
static main(args) {
@@ -163,9 +163,9 @@ class AnalyticsApiLogPerformanceBatch extends BaseBatch {
}
static void sendEmail(String sub, String msg) {
jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
// jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
jms.sendMail("wangwenyan@fangjia.com", sub, msg)
jms.sendMail("helvyuan@fangjia.com", sub, msg)
// jms.sendMail("helvyuan@fangjia.com", sub, msg)
jms.sendMail("gaojuan@fangjia.com", sub, msg)
}
@@ -32,11 +32,11 @@ class CommonLogAnaBatch extends BaseBatch {
def static bjEvaReport = [];
public static final def EMAILS = [
"jiangxiaochun@fangjia.com",
"limingmin@fangjia.com",
"xiaohai@fangjia.com",
"luxuping@fangjia.com",
"yukai@fangjia.com"
"jiangxiaochun@fangjia.com"
// "limingmin@fangjia.com",
// "xiaohai@fangjia.com",
// "luxuping@fangjia.com",
// "yukai@fangjia.com"
];
static main(args) {
@@ -26,12 +26,12 @@ class DaikuanLogAnaBatch extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public static final def EMAILS = [
"jiangxiaochun@fangjia.com",
"limingmin@fangjia.com",
"zhongjingyi@fangjia.com",
"xiaohai@fangjia.com",
"luxuping@fangjia.com",
"yukai@fangjia.com"
"jiangxiaochun@fangjia.com"
// "limingmin@fangjia.com",
// "zhongjingyi@fangjia.com",
// "xiaohai@fangjia.com",
// "luxuping@fangjia.com",
// "yukai@fangjia.com"
];
static main(args) {
@@ -148,14 +148,14 @@ class ReadBkoffSafeLogBatch extends BaseBatch{
}
static void sendEmail(String sub, String msg) {
jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
jms.sendMail("joe@fangjia.com", sub, msg)
jms.sendMail("xiaohai@fangjia.com", sub, msg)
// jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
// jms.sendMail("joe@fangjia.com", sub, msg)
// jms.sendMail("xiaohai@fangjia.com", sub, msg)
jms.sendMail("caoyuyan@fangjia.com", sub, msg)
jms.sendMail("wangchencen@fangjia.com", sub, msg)
jms.sendMail("yaojingmeng@fangjia.com", sub, msg)
jms.sendMail("chichilook@qq.com", sub, msg)
jms.sendMail("sunjian@fangjia.com", sub, msg)
// jms.sendMail("wangchencen@fangjia.com", sub, msg)
// jms.sendMail("yaojingmeng@fangjia.com", sub, msg)
// jms.sendMail("chichilook@qq.com", sub, msg)
// jms.sendMail("sunjian@fangjia.com", sub, msg)
}
static public String getCatById(String id) {
@@ -272,4 +272,25 @@ class InsidePolygon {
}
return flag;
}
public static def isInArea2(lng, lat, pointMap){
def flag = false;
Set<String> keys = pointMap.keySet();
def name = null
for(String key :keys){
def prints = pointMap.get(key)
for(def pointArr : prints){
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
if(flag){
def arrs = key.split("##")
name = arrs[1];
break;
}
}
if(flag){
break;
}
}
return name;
}
}
@@ -0,0 +1,175 @@
package com.fangjia.script.majintao
import com.fangjia.core.RegionData
import com.fangjia.core.ProvinceData
import com.fangjia.core.util.MongoUtils
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
def map = [
["学校", "高等院校", "中学", "小学", "幼儿园", "成人教育", "职业技术学校"]: "学校",
["综合医院", "专科医院"] : "医院",
["银行"] : "银行",
["超级市场", "家电电子卖场", "便民商店/便利店"] : "超市",
["商场"] : "商场",
["美容美发店"] : "美容院",
["健身中心"] : "健身中心",
["公园广场", "公园"] : "公园/绿地",
["公交车站"] : "公交",
["飞机场"] : "机场",
["火车站"] : "火车站",
["快餐厅", "外国餐厅", "中餐厅"] : "餐馆",
["宾馆酒店", "旅馆招待所"] : "酒店"
] as HashMap<List<String>, String>
def type3In = ["学校", "高等院校", "中学", "小学", "幼儿园", "成人教育", "职业技术学校", "健身中心", "公园广场", "公园", "地铁站", "飞机场", "火车站"]
// 输出 Map 的内容
map.each { key, value ->
println("$key: $value")
}
def provinceList = []
def provinceMap = ProvinceData.data
provinceMap.each { pro ->
provinceList.add(pro.getValue()[0])
}
def cityMap = ProvinceData.GOVERNMENT_CITY_LIST
cityMap.each { city ->
provinceList.add(city.getValue()[0])
}
def specialMap = ProvinceData.SPECIAL_ADMIN_REGION
specialMap.each { spe ->
provinceList.add(spe.getValue()[1])
}
def pcmap = [:]
provinceList.remove("广西省")
provinceList.add("广西壮族自治区")
DSCollection seabird = new DataStore("192.168.0.43", 28017).getCollection("fangjia_base.baidu_place_poi2")
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "192.168.0.43", 28017)
//def type3Not = ["学校内部设施", "进港", "出港", "候船室", "住宿服务相关"]
//
//def type2Not = ["学校", "专科医院", "综合医院", "诊所", "银行", "自动提款机", "便民商店/便利店", "家电电子卖场", "超级市场", "商场", "美容美发店", "地铁站", "港口码头", "住宿服务"]
//def type2In = ["运动场馆", "公园广场", "公交车站", "机场相关", "火车站", "餐饮服务"]
// 运动场馆 且为 健身中心
// 公园广场 公园广场
//公园广场 公园
// 公交车站 公交车站相关
// 机场相关 飞机场
// 火车站 火车站
// 餐饮服务 快餐厅 外国餐厅 中餐厅
println(provinceList)
//provinceList = provinceList.subList(0,1)
def c = 0
def citys = []
def citys2 = []
for (p in provinceList) {
println(p)
def result = []
ElasticUtils.executeSqlQuery("SELECT location,name,adname,cityname,m_date,id,address,type FROM amap_poi WHERE pname = '" + p + "'", { Map data ->
// println(data["type"] + "-----")
c = c+1
if (c%10000==0){
println("已处理$c 条")
}
def locationString = String.valueOf(data.get("location"))
def typeString = String.valueOf(data.get("type"))
def cityString = String.valueOf(data.get("cityname"))
// citys2.add(cityS)
// print(cityString)
String city
if ("台湾省".equals(cityString) | "香港特别行政区".equals(cityString) | "澳门特别行政区".equals(cityString)) {
city = cityString
} else {
city = RegionData.formatCity(cityString)
}
def m_date = data["m_date"]
def region = data["adname"]
region = RegionData.formatRegion(city,region)
def address = String.valueOf(data.get("address"))
def uid = String.valueOf(data.get("id"))
def name = data.get("name")
if (typeString.contains("|")) {
typeString = typeString.split("'|'")[0]
}
// println(typeString)
def secondType = typeString.split(";")[1]
def smallType = typeString.split(";")[2]
// println(smallType.getClass().getName())
// 使用正则表达式匹配经度和纬度
def matcher = locationString =~ /(-?\d+\.\d+)/
def keyword = ""
def flag = 0
// 类型映射
// 用三级分类判断
if (smallType in type3In) {
// 遍历Map的键集合,检查目标字符串是否存在于键集合中
map.keySet().each { key ->
if (key.contains(smallType)) {
def value = map[key]
// println("$smallType 存在于Map的键中,对应的值为: $value")
keyword = value
}
}
} else {
//用二级分类判断
map.keySet().each { key ->
if (key.contains(secondType)) {
def value = map[key]
// println("$secondType 存在于Map的键中,对应的值为: $value")
keyword = value
}
}
}
// 获取匹配到的经度和纬度值
def lng = matcher[0][0]
lng = lng.toDouble()
def lat = matcher[1][0]
lat = lat.toDouble()
// println "纬度: $lng, 经度: $lat,"
if (!"".equals(keyword)) {
def dbo = new BasicDBObject()
dbo["_id"] = new ObjectId()
dbo["keyword"] = keyword
dbo["lat"] = lat
dbo["name"] = name
dbo["city"] = city
dbo["m_date"] = m_date
dbo["lng"] = lng
dbo["c_date"] = m_date
dbo["address"] = address
dbo["uid"] = uid
dbo["status"] = 0
dbo["source"] = "gaode"
dbo["region"] = region
result.add(dbo)
}
// print(result)
// println("888")
})
if (null != result && result.size() > 0) {
m43.insert(result)
}
println("已处理完$p 的数据")
}
@@ -0,0 +1,25 @@
package com.fangjia.script.majintao
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
def count = 0
//ElasticUtils.executeSqlQuery("SELECT * FROM seawater WHERE city = '重庆' and region = '渝北' and district_name = '棕榈泉国际花园四期'and p_date > 20230801 ",{Map doc->
ElasticUtils.executeSqlQuery("SELECT * FROM seawater WHERE p_date > 20230801 order by p_date desc limit 100",{Map doc->
def keys = ["c_date","_id","cat","city","region","name","address","block","district_name","total_price","area","height","floor","fitment","estate_type","hall","toilet","publish_date","url","direction","build_type","pay_type","line","rent_type","status","room"]
// def dbo = new BasicDBObject()
// for( String key in keys){
// dbo[key] = doc[key]
// }
// println(doc['m_date'])
// print(doc['name'])
if (count==0)print(doc)
count+=1
// DbUtils.seabird.save(dbo)
})
//DbUtils.seabird.find(new BasicDBObject("district_name","棕榈泉国际花园四期")).each {
// println(it)
//}
print(count)
@@ -0,0 +1,57 @@
import com.fangjia.core.ProvinceData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import com.lowagie.text.Row
import javafx.scene.control.Cell
import jxl.Sheet
import jxl.Workbook
import org.apache.poi.ss.usermodel.WorkbookFactory
//String excelFilePath = "F:\\估家\\文件\\安徽省联社\\全国省级行政区房屋覆盖情况表 - 副本.xlsx"
//ExcelUtils.removeEmptyRow(excelFilePath)
//ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
//
// province1 = items['A']
// if (!province1.equals('黑龙江省') || !province1.equals('内蒙古自治区')) {
// province = province1.substring(0, 2)
// } else {
// province = province1.substring(0, 3)
// }
//
// println('province: ' + province)
//
// def citys = ProvinceData.getAllCity(province)
// println(citys)
//// if (citys.size() > 0) {
//// ExcelUtils.writeToExcel(excelFilePath, citys, province)
//// }
//
//})
//
//import com.fangjia.core.ProvinceData
//
//import com.fangjia.script.util.ExcelUtils
//
//
//String excelFilePath = "F:\\估家\\文件\\安徽省联社\\全国省级行政区房屋覆盖情况表.xlsx"
//ExcelUtils.removeEmptyRow(excelFilePath)
//ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
//
// def province1 = items['A']
// def province
//
// if (!province1.equals('黑龙江省') && !province1.equals('内蒙古自治区')) {
// province = province1.substring(0, 2)
// } else {
// province = province1.substring(0, 3)
// }
//
// println('province: ' + province)
//
// def citys = ProvinceData.getAllCity(province)
// println(citys)
// items['D'] = citys.toString()
//
//})
@@ -0,0 +1,72 @@
package com.fangjia.script.majintao
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
String excelFilePath = "F:\\估家\\文件\\北交所\\testcopy1.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
String comprehensiveUrl = "http://localhost:9180/comprehensiveQuery/houseValueInformation";
String communityUrl = "http://open.fangjia.com/addrSvc/avgPrice";
String valueUrl = "http://open.fangjia.com/property/addressEvaluate";
String caseUrl = "http://open.fangjia.com/house/case";
String poiUrl = "http://open.fangjia.com/district/addrFacilitys";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
String city = items['A']
String address = items['B']
double size = items['C']
System.out.println(city + "---" + address + "----" + size);
map.put("city", city)
map.put("address", address)
map.put("size", size.toString())
map.put("strctureArea", size.toString())
map.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB")
if (!address.contains("%")) {
// 发送HTTP请求
long start
long end
// start = System.currentTimeMillis();
// httpRequest.getJsonContent(communityUrl, map, "UTF-8")
//
//
// end = System.currentTimeMillis();
// items['D'] = String.valueOf(end - start);
//
// start = System.currentTimeMillis();
// httpRequest.getJsonContent(valueUrl, map, "UTF-8")
// end = System.currentTimeMillis();
// items['E'] = String.valueOf(end - start);
//
// start = System.currentTimeMillis();
// httpRequest.getJsonContent(caseUrl, map, "UTF-8")
// end = System.currentTimeMillis();
// items['F'] = String.valueOf(end - start);
//
// start = System.currentTimeMillis();
// httpRequest.getJsonContent(poiUrl, map, "UTF-8")
// end = System.currentTimeMillis();
// items['G'] = String.valueOf(end - start);
start = System.currentTimeMillis();
def content = httpRequest.getJsonContent(comprehensiveUrl, map, "UTF-8")
end = System.currentTimeMillis();
items['H'] = String.valueOf(end - start);
println content.get("msg")
if(content.get("msg").contains("该地址不支持估价!")) {
items['I'] = "该地址不支持估价!"
}
}
})
@@ -0,0 +1,72 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.ExcelUtils
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count = count + 1
if (count % 10 == 0) {
println("已处理:" + count)
}
sleep(50)
def address = items['A']
def map = SignatureUtils.getAliAddressStrc(address)
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
if ("prov" in map) {
def province = map.get("prov")
items['D'] = province
}
if ("city" in map) {
def city = map.get("city")
items['H'] = city
}
if ("district" in map) {
def district = map.get("district")
items['L'] = district
}
if ("town" in map) {
def town = map.get("town")
items['P'] = town
}
if ("community" in map) {
def community = map.get("community")
items['T'] = community
}
// if ("road" in map) {
// def road = map.get("road")
// items['G'] = road
//
// }
// if ("poi" in map) {
//
// def poi = map.get("poi")
// items['H'] = poi
//
// }
// if ("roomno" in map) {
// def roomno = map.get("roomno")
// items['I'] = roomno
//
// }
// if ("devzone" in map) {
// def devzone = map.get("devzone")
// items['J'] = devzone
//
// }
// if ("roadno" in map) {
// def roadno = map.get("roadno")
// items['K'] = roadno
// }
})
@@ -0,0 +1,76 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.ExcelUtils
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count = count + 1
if (count % 10 == 0) {
println("已处理:" + count)
}
sleep(50)
def address = items['A']
def pro = items['D']
if (pro == null) {
def map = SignatureUtils.getAliAddressStrc(address)
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
if ("prov" in map) {
def province = map.get("prov")
items['D'] = province
}
if ("city" in map) {
def city = map.get("city")
items['H'] = city
}
if ("district" in map) {
def district = map.get("district")
items['L'] = district
}
if ("town" in map) {
def town = map.get("town")
items['P'] = town
}
if ("community" in map) {
def community = map.get("community")
items['T'] = community
}
// if ("road" in map) {
// def road = map.get("road")
// items['G'] = road
//
// }
// if ("poi" in map) {
//
// def poi = map.get("poi")
// items['H'] = poi
//
// }
// if ("roomno" in map) {
// def roomno = map.get("roomno")
// items['I'] = roomno
//
// }
// if ("devzone" in map) {
// def devzone = map.get("devzone")
// items['J'] = devzone
//
// }
// if ("roadno" in map) {
// def roadno = map.get("roadno")
// items['K'] = roadno
// }
}
})
@@ -0,0 +1,64 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\ÁÙʱÊý¾Ý\\2.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
String url = "http://192.168.0.19:31080/FangJiaAddressService/v3/structure";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
String address = items['A']
def map = [:]
map.put("address", address)
map.put("complete", "true")
map.put("addressAssessment", "true")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if ("province" in result) {
items['B'] = result.get("province")
}
if ("provinceCode" in result) {
items['C'] = result.get("provinceCode")
}
if ("city" in result){
items['D'] = result.get("city")
}
if("cityCode" in result){
items['E'] = result.get("cityCode")
}
if ("district" in result){
items['F'] = result.get("district")
}
if ("districtCode" in result){
items['G'] = result.get("districtCode")
}
if ("township" in result){
items['H'] = result.get("township")
}
if ("townshipCode" in result){
items['I'] = result.get("townshipCode")
print(result.get("townshipCode"))
}
if ("address" in result){
items['J'] = result.get("address")
}
if ("formatAddress" in result){
items['K'] = result.get("formatAddress")
}
})
@@ -0,0 +1,124 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
count += 1
if (count % 10 == 0) {
println("已处理:" + count)
}
String address = items['A']
String province = items['B']
if (province == null) {
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
def map = [:]
map.put("address", address)
map.put("complete", "true")
map.put("addressAssessment", "true")
map.put("type", "1")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if (result != null) {
if ("province" in result) {
items['B'] = result.get("province")
}
if ("provinceCode" in result) {
items['C'] = result.get("provinceCode")
}
if ("city" in result) {
items['D'] = result.get("city")
}
if ("cityCode" in result) {
items['E'] = result.get("cityCode")
}
if ("district" in result) {
items['F'] = result.get("district")
}
if ("districtCode" in result) {
items['G'] = result.get("districtCode")
}
if ("town" in result) {
items['H'] = result.get("town")
}
if ("townCode" in result) {
items['I'] = result.get("townCode")
}
if ("committee" in result) {
items['J'] = result.get("committee")
}
if ("committeeCode" in result) {
items['K'] = result.get("committeeCode")
}
if ("typeCode" in result) {
items['L'] = result.get("typeCode")
}
if ("typeName") {
items['M'] = result.get("typeName")
}
if ("verifyFlag") {
items['N'] = result.get("verifyFlag")
}
if ("addrVerify") {
items['O'] = result.get("addrVerify")
}
if ("countyLevelCity") {
items['P'] = result.get("countyLevelCity")
}
if ("countyGov") {
items['Q'] = result.get("countyGov")
}
if ("communitySource") {
items['R'] = result.get("communitySource")
}
if ("administrativeTowns" in result) {
items['T'] = result.get("administrativeTowns")
}
if ("devDistrict" in result) {
items['V'] = result.get("devDistrict")
}
if ("devDistrictName" in result) {
items['W'] = result.get("devDistrictName")
}
if ("splitCommittee" in result) {
items['X'] = result.get("splitCommittee")
}
if ("splitStreet" in result) {
items['Y'] = result.get("splitStreet")
}
if ("poiName" in result) {
items['Z'] = result.get("poiName")
}
if ("splitStreetNum" in result) {
items['AA'] = result.get("splitStreetNum")
}
if ("dev" in result) {
items['AB'] = result.get("dev")
}
}
}
})
@@ -0,0 +1,147 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\对公.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count+=1
if (count%10==0){
println("已处理:"+count)
}
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
String address = items['A']
// print(address)
// province;
// city;
// district;
// town;
// committee;
// type;
// provinceCode;
// cityCode;
// districtCode;
// townCode;
// committeeCode;
// typeCode;
// typeName;
// verifyFlag;//涉农标识
// addrVerify;//地址是否有效信息
// countyLevelCity;//是否为县级市
// administrativeTowns;//是否市辖建制镇
// countyGov;//是否为县政府
// devDistrict;//是否为开发区
// devDistrictName;//开发区
// splitCommittee;
// splitStreet;
// poiName;
// splitStreetNum;
// dev;
// communitySource;//居委来源
def map = [:]
map.put("address", address)
map.put("complete", "true")
map.put("addressAssessment", "true")
map.put("type", "2")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if ("province" in result) {
items['B'] = result.get("province")
}
if ("provinceCode" in result) {
items['C'] = result.get("provinceCode")
}
if ("city" in result){
items['D'] = result.get("city")
}
if("cityCode" in result){
items['E'] = result.get("cityCode")
}
if ("district" in result){
items['F'] = result.get("district")
}
if ("districtCode" in result){
items['G'] = result.get("districtCode")
}
if ("town" in result){
items['H'] = result.get("town")
}
if ("townCode" in result){
items['I'] = result.get("townCode")
}
if ("committee" in result){
items['J'] = result.get("committee")
}
if ("committeeCode" in result){
items['K'] = result.get("committeeCode")
}
if ("typeCode" in result){
items['L'] = result.get("typeCode")
}
if ("typeName"){
items['M'] = result.get("typeName")
}
if ("verifyFlag"){
items['N'] = result.get("verifyFlag")
}
if ("addrVerify"){
items['O'] = result.get("addrVerify")
}
if ("countyLevelCity"){
items['P'] = result.get("countyLevelCity")
}
if ("countyGov"){
items['Q'] = result.get("countyGov")
}
if ("communitySource"){
items['R'] = result.get("communitySource")
}
if ("administrativeTowns" in result){
items['T'] = result.get("administrativeTowns")
}
if ("devDistrict" in result){
items['V'] = result.get("devDistrict")
}
if ("devDistrictName" in result){
items['W'] = result.get("devDistrictName")
}
if ("splitCommittee" in result){
items['X'] = result.get("splitCommittee")
}
if ("splitStreet" in result){
items['Y'] = result.get("splitStreet")
}
if ("poiName" in result){
items['Z'] = result.get("poiName")
}
if ("splitStreetNum" in result){
items['AA'] = result.get("splitStreetNum")
}
if ("dev" in result){
items['AB'] = result.get("dev")
}
})

Some files were not shown because too many files have changed in this diff Show More