Merge branch '小区周边POI定时任务' into script_develop

# Conflicts:
#	projects/fangjia-script/pom.xml
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy
#	projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy
This commit is contained in:
sjshenjian
2021-06-21 09:16:32 +08:00
41 changed files with 2421 additions and 150 deletions
+6 -4
View File
@@ -39,13 +39,15 @@ spec:
name: web
protocol: TCP
readinessProbe:
tcpSocket:
port: 8080
httpGet:
path: /k8s/readiness
port: web
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 8080
httpGet:
path: /k8s/liveness
port: web
initialDelaySeconds: 10
periodSeconds: 5
volumeMounts:
@@ -0,0 +1,28 @@
package com.fangjia.bkoff.controllers;
/**
* 房价网后台存活监控
*
* @author shenjian
* @since 2021/6/7
*/
public class K8sController {
/**
* 存活检查
*
* @return
*/
public Object livenessAction() {
return "liveness";
}
/**
* 就绪检查
*
* @return
*/
public Object readinessAction() {
return "readiness";
}
}
@@ -40,7 +40,8 @@ public class AuthInterceptor implements Interceptor {
request.setAttribute("__resources", request.getSession().getAttribute(userId + "_resources"));
if (invocation.getAction().getController().getName().equals("login")
|| invocation.getAction().getController().getName().equals("")) {
|| invocation.getAction().getController().getName().equals("")
|| invocation.getAction().getController().getName().equals("k8s")) {
return invocation.invoke();
}
@@ -237,7 +237,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
}
private JSONObject query(String id,String category, String city, String region, String name, List<String> alias,
List<String> regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){
List<String> regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice,
Integer room, String root,String sort, boolean order, int start, int limit,String table){
long time1 = System.currentTimeMillis();
JSONObject reqBody = new JSONObject();
JSONObject q = new JSONObject();
@@ -351,7 +352,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
if(endPrice != null){
temp2.put("lte", endPrice);
}
param.put("total_price", temp2);
param.put("avg_price", temp2);
temp.put("range", param);
jsonList.add(temp);
}
@@ -156,12 +156,12 @@ public class GuzhiController {
return mv;
}
//福州农商行人工录入
public Object checkAction(@Valid GuzhiCheckModel model){
RestfulModelAndView mv = new RestfulModelAndView();
mv.setModel(BasicModel.buildOKModel());
String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()
String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()+" 区域:"+model.getDistrict() +" 小区名称:"+(model.getName()!=null?model.getName():"")
+ " 产证地址:" + model.getAddress();
DingDingMessageUtil.sendTextMessage("065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60", msg);
return mv;
@@ -671,7 +671,7 @@ public class PropertyController {
&& !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId)
&& !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId)
&& !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId)
&& !"1007747".equals(userId)
&& !"1007747".equals(userId) && !"1007757".equals(userId)
&& model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制
mav.setModel(BasicModel.buildUnAuthorizedModel());
return mav;
@@ -284,6 +284,8 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
private String user;
private Integer avgPrice;
public void setDecorationAge(int decorationAge) {
this.decorationAge = decorationAge;
@@ -895,4 +897,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
public void setAtticSize(Double atticSize) {
this.atticSize = atticSize;
}
public Integer getAvgPrice() {
return avgPrice;
}
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
}
@@ -30,12 +30,12 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
void process(Object args) {
List<String> cities = [];
def allCity = CommonUtils.findAllCity()
allCity = ["ËÕÖÝ"]
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
cities.addAll(allCity)
def districts = []
cities.each {def city->
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0).append("name", "ÎáÔù㳡"), keys).batchSize(2000).toList()
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList()
println(city)
}
ExecutorService executorService = Executors.newFixedThreadPool(20)
@@ -54,6 +54,10 @@ class DealPriceMathBatch extends BaseBatch{
def flag = true;
def startDate = DateUtil.getFirstDayOfMonth(new Date(),-1)
def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
cityMap.each{city,alias->
println "开始加载【${city}】小区"
if(city != "昆玉"){
@@ -61,10 +65,8 @@ class DealPriceMathBatch extends BaseBatch{
districtMap.putAll(detailMap)
}
}
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
// def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
// def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
while(flag){
println "开始加载解析数据......"
@@ -211,23 +211,35 @@ class AccessLogBatch extends BaseBatch{
}
def read(){
String fileName = "C:\\Users\\gaojuan\\Desktop\\localhost_access_log.2019-06-01.txt";
String fileName = "D:\\Êý¾Ý\\logs\\performance.open.log";
File file = new File(fileName);
def rets = []
file.eachLine{
def index = 0
def list =[]
def slowCount = 0
file.eachLine{it->
if(it.contains("/fangjiatong2/")){
try{
it = it.replace(it, URLDecoder.decode(URLDecoder.decode(it, "utf-8"), "UTF-8"))
}catch(IllegalArgumentException e){}
if(it.contains("ws/czbank") && it.contains("entity:House")){
// println it
index++
// try{
def s1 = it.substring(it.indexOf("time:")+5, it.length())
s1 = s1.substring(0, s1.indexOf("\t"))
list << s1.toInteger()
if(s1.isNumber() && s1.toInteger()>=4000){
println it
slowCount++
}
// }catch(IllegalArgumentException e){}
}
if(!it.contains("äîϪËÄ´å")){
rets << it
}
}
writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt");
println list.sum()/list.size()
println index
println "slowCount:"+slowCount
// writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt");
}
}
@@ -102,7 +102,7 @@ class DistrictInfoExportBatch extends BaseBatch{
row << rMap.estate_company
row << getBuildStructure(rMap.build_structure)//为空的,默认钢混
row << rMap.loudong_count
row << processAlias(rMap.alias)
row << processAlias(rMap.alias, rMap.addr_alias)
def addr = fzAddrMap.get(rMap.get("_id").toString());
if(addr){
@@ -110,20 +110,32 @@ class DistrictInfoExportBatch extends BaseBatch{
}else{
row << rMap.address
}
//row << processAlias(rMap.addr_alias)
result << row
}
}
return result
}
def processAlias(alias){
def processAlias(alias, def addr_alias){
def newAlias = []
if(alias){
def newAlias = []
for(def name : alias){
if(name && !name.contains("(") && !name.contains("¡¢") && !name.contains("?") && !name.contains("£¨")){
if(name && !name.contains("(") && !name.contains("¡¢") && !name.contains("?") && !name.contains("£¨") && !newAlias.contains(name)){
newAlias << name
}
}
}
if(addr_alias){
for(def name : addr_alias){
if(name && !name.contains("(") && !name.contains("¡¢") && !name.contains("?") && !name.contains("£¨") && !newAlias.contains(name)){
newAlias << name
}
}
}
if(newAlias){
return newAlias.join(",");
}
return null;
@@ -165,6 +177,9 @@ class DistrictInfoExportBatch extends BaseBatch{
rets << "非钢混"
}
}
if(rets.size()==0){
rets << "¸Ö»ì"
}
return rets.join(",")
}
return "钢混";
@@ -76,8 +76,8 @@ class DistrictPriceExportBatch extends BaseBatch{
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
def lastDate = new Date()
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
def result = []
for(def rMap : districts){
@@ -0,0 +1,133 @@
package com.fangjia.script.fuzhou
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
def count0 = 0
def count1 = 0
def count2 = 0
def count3 = 0
def count4 = 0
def count5 = 0
def count6 = 0
def count7 = 0
def count8 = 0
def total=4943
for(def d : datas){
def list = []
if(d['G']){
list << d['G']
}
if(d['H']){
list << d['H']
}
if(d['I']){
list << d['I']
}
if(d['J']){
list << d['J']
}
if(d['K']){
list << d['K']
}
if(d['L']){
list << d['L']
}
if(d['M']){
list << d['M']
}
if(d['N']){
list << d['N']
}
if(d['O']){
list << d['O']
}
if(list.size()==0){
count0++
}
if(list.size()==1){
count1++
}
if(list.size()==2){
count2++
}
if(list.size()==3){
count3++
}
if(list.size()==4){
count4++
}
if(list.size()==5){
count5++
}
if(list.size()==6){
count6++
}
if(list.size()==7){
count7++
}
if(list.size()==8){
count8++
}
}
println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total)))
println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total)))
println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total)))
println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total)))
println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total)))
println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total)))
println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total)))
println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total)))
println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total)))
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
//def total = 8018
//def names = ["诸葛找房":"zhugezhaofang_esf_price","安居客":"anjuke_esf_price","搜房":"soufang_esf_price","链家":"lianjia_esf_price","城市房产":"cityhouse_esf_price","房探":"fangtan_esf_price","房多多":"fangdoudou_esf_price","麦田":"maitian_esf_price", "58同城":"58tc_esf_price"]
//
//def rets = []
//def sDate = 202011
//rets << ["时间","诸葛找房","安居客","搜房","链家","城市房产","房探","房多多","麦田", "58同城"]
//def ret = []
//ret << sDate
//names.each{k,v->
// def crawlCount = crawlCount(k, sDate)
// def count = count(v, sDate)
// println k +" "+DataHandleUtils.getDoubleRound(2, count*100.0/total)+"("+crawlCount+")"
// ret << DataHandleUtils.getDoubleRound(2,count*100.0/total)+"("+crawlCount+")"
//}
//rets.add(ret)
//ExcelUtils.write("d:\\数据\\第三方均价覆盖率1.xlsx",rets,"数据")
//
//def crawlCount(k, sDate){
// def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
// def q = new BasicDBObject()
// q.put("city","福州")
// q.put("s_date", sDate)
// q.put("source",k)
// return thirdPartyColl.count(q)
//}
//
//
//def count(name, sDate){
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
// def q = new BasicDBObject()
// q.put("city","福州")
// q.put("s_date", sDate)
// q.put(name, new BasicDBObject('$gt', 0));
// return AVERAGE_STATISTICS.count(q)
//}
@@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils
*/
class ThirdPriceExportBatch extends BaseBatch{
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
retMap.cityhouse_esf_price = d.cityhouse_esf_price
}
if(!retMap.fangtan_esf_price && d.fangtan_esf_price){
retMap.fangtan_esf_price = d.fangtan_esf_price
if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取
retMap.fangtan_esf_price = d.jiwu_esf_price
}
if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){
retMap.fangdoudou_esf_price = d.fangdoudou_esf_price
if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多
retMap.fangdoudou_esf_price = d.get("365house_esf_price")
}
if(!retMap.maitian_esf_price && d.maitian_esf_price){
@@ -3,6 +3,9 @@ package com.fangjia.script.gj
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.paimai.GaoDeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.JacksonUtil
@@ -30,39 +33,40 @@ import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
class GaoDeMapCrawl {
@BatchOwner(Watcher.GaoJuan)
class GaoDeMapCrawl extends BaseBatch{
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
static def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
public static final def dbUrl = "mongodb://192.168.0.235:27018";
final def dbUrl = "mongodb://192.168.0.235:27018";
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
final AtomicInteger count = new AtomicInteger()
def updateLists = []
def updateErrorLists = []
def flagLists = []
def index = 0
def prefix = "district_"
// def categorys = ["141202", "141203"]//中学,小学
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def prefix = "district_"
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def categorys = ["190301"]//道路相关
def prefix = "village_"
def categorys = ["190108","190109"]//村庄
def totalCount = new AtomicLong()
static main(args) {
def task = new GaoDeMapCrawl();
// def citys = CommonUtils.findAllCity()
// def i =1
// for(def city : citys){
// i++
// println i+" "+ city
// }
task.run()
// task.allRun();
runBatch(new GaoDeMapCrawl(), args)
}
@Override
void process(Object args) {
run()
}
def allRun(){
def cityLations = null;
def stop = false;
@@ -110,14 +114,20 @@ class GaoDeMapCrawl {
def run(){//,"西安","重庆"
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
def cityLations = findAllLation(city);
println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations);
def stop = false
while(stop == false){
def cityLations = findAllLation(city);
if(!cityLations){
stop = true;
}else{
println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations);
}
}
}
}
@@ -129,7 +139,7 @@ class GaoDeMapCrawl {
q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false))
// q.put("status", "失败")
// q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false))
def results = BOUNDS_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
return results;
}
@@ -142,7 +152,7 @@ class GaoDeMapCrawl {
// void run() {
// try{
// totalCount.addAndGet(1)
searching(location)
searching(i, location)
// }catch(Exception e){
// e.printStackTrace()
// } finally {
@@ -169,7 +179,7 @@ class GaoDeMapCrawl {
}
}
def searching(location){
def searching(i, location){
// def bound_gd = location.bound_gd
// def lats = [bound_gd[1], bound_gd[3]]
// def lngs = [bound_gd[0], bound_gd[2]]
@@ -177,7 +187,7 @@ class GaoDeMapCrawl {
// lngs.sort()
// println index
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
search(location)
search(i, location)
// }
if(updateLists.size() > 0 && updateLists.size()>1000){
@@ -196,7 +206,7 @@ class GaoDeMapCrawl {
}
def search(location){
def search(i, location){
def bound_gd = location.bound_gd
def lats = [bound_gd[1], bound_gd[3]]
def lngs = [bound_gd[0], bound_gd[2]]
@@ -230,7 +240,7 @@ class GaoDeMapCrawl {
updateErrorLists.addAll(location)
}
// update(location);
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
println "[ "+i+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
}
def processData(rets){
@@ -43,6 +43,24 @@ class OpenApiUtils {
}
static def matchDetailDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
address = address.replace("£¨","(").replace("£©",")").replace(" ","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
}
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dto = doc.result
return dto
}
}
return null;
}
static def matchDistrict2(def city, def address, def preciseLevelflag){
address = address.replace("£¨","(").replace("£©",")").replace(" ","").replace("#","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-03-01"
static def statDate = "2021-04-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -0,0 +1,77 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CcbCityPriceStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new CcbCityPriceStat()
task.run()
}
def run(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def rets = []
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
def cityRets = []
for(def dist : districts){
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
}
}
print city +" "+ cityRets.size()
rets.addAll(cityRets)
}
println "总量:"+rets.size()
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
}
@@ -0,0 +1,322 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import com.fangjia.script.gj.*
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataMatch {
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
static main(args) {
def task = new DataMatch()
// task.run1()
task.run2()
// task.run3()
// println task.findAddress("昌州街道棠香路")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
}
def run1(){
def fileName = "D:\\数据\\重庆农商行\\估值2-V5.xlsx"
//matchRegionValue(fileName);
matchDistrict(fileName)
// ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['O']=="新增"){
//
// d['L'] = d['P']
// d['M'] = d['Q']
// d['N'] = d['R']
// }
// },0)
}
def parseAddress(def fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
},0)
}
def matchDistrict(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def region = d['Q']
def city = d['P']
// if(!d['I'] && !d['J']){
// city = d['B']
// region = d['C']
// }
if(d['R'] && !d['G']){
def results = null
// if(d['E'] && isStreet(d['E'])){//加上效果并没有想象中的好
// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D'])
// }
if((results && results.precise<0) || !results){
results = OpenApiClient.addressMatch(city, d['R'], "property", false, region, null)
}
// if((d['E'] && results && results.precise<0) || !results){
// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D'])
// }
if(results && results.precise>=0){
d['K'] = results.city
d['L'] = results.district
d['M'] = results.name
d['N'] = results.address
}
}
},0);
}
def run2(){
def fileName = "D:\\数据\\重庆农商行\\估值2-V2.xlsx"
// matchRegionValue(fileName);
matchDistrict2(fileName)
}
def matchDistrict2(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def region = d['I']
def city = d['J']
if(!d['I']){
city = d['J']
}
if(city && d['E'] && !d['O'] ){
def addressResult = OpenApiClient.addressStandard(city, d['E'])
if(addressResult){
if(addressResult.street && addressResult.streetNumber){
d['H'] = addressResult.street+addressResult.streetNumber
}
}
def results = null
if(d['H']){
results = OpenApiClient.addressMatch(city, d['H'], "property", false, region, null)
}
if((d['E'] && results && results.precise<0) || !results){
results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, null)
}
if(results && results.precise>=0){
d['K'] = results.city
d['L'] = results.district
d['M'] = results.name
d['N'] = results.address
}
}
},0);
}
def run3(){
def fileName = "D:\\数据\\重庆农商行\\估值3.xlsx"
// matchDistrict3(fileName)
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['I']){
d['I'] = d['I'].toString().replace("省","").replace("市","")
}
if(d['J'] && d['I']){
d['J'] = RegionData.formatRegion(d['I'], d['J'])
}
},0)
}
def matchDistrict3(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def address = d['B']
if(address) {
def arrs = address.split("->")
def city = null
def region = null
def addr = null
if (arrs && arrs.size() == 4) {
city = arrs[0]
region = arrs[2]
addr = arrs[3]
d['I'] = city
d['J'] = region
} else if (arrs && arrs.size() == 2) {
def parseCity = CommonUtils.parseCityInAddress(arrs[1])
city = parseCity
d['I'] = parseCity
addr = arrs[1]
}
if (addr && !d['O']) {
def addressResult = OpenApiClient.addressStandard(city, addr)
if (addressResult) {
if (!region) {
region = addressResult.district
d['J'] = region
}
if (addressResult.street && addressResult.streetNumber) {
d['H'] = addressResult.street + addressResult.streetNumber
}
}
def results = null
def name = null
if (d['C']) {
name = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
}
if (name) {
results = OpenApiClient.addressMatch(city, name, "property", false, region, null)
}
if ((addr && results && results.precise < 0) || !results) {
results = OpenApiClient.addressMatch(city, addr, "property", false, region, name)
}
if (results && results.precise >= 0) {
d['K'] = results.city
d['L'] = results.district
d['M'] = results.name
d['N'] = results.address
}
}
}
},0);
}
def matchRegionValue(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def code = d['C']
if(code && code.toString().isNumber()){
if(code.toInteger().toString()=="500226"){
d['I'] = "荣昌"
d['J'] = "重庆"
}else if(code.toInteger().toString()=="500223"){
d['I'] = "潼南"
d['J'] = "重庆"
}else{
def data = findRegionCodeValue(code.toInteger().toString());
if(data){
d['I'] = data.fangjia_district
d['J'] = data.fangjia_city
}else{
data = findCityCodeValue(code.toInteger().toString())
if(data){
d['J'] = data.fangjia_city
}
}
}
}
},0)
}
def findCityCodeValue(value){
def q = new BasicDBObject()
q.put("city_code", value)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def count = dbcursor.count()
if(count >1){
def data = null
for(def db : dbcursor){
data = db
break
}
return data;
}
}
def findRegionCodeValue(value){
def q = new BasicDBObject()
q.put("district_code", value)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def count = dbcursor.count()
if(count == 1){
def data = null
for(def db : dbcursor){
data = db
break
}
return data;
}
}
def findAddress(String address) {
//Pattern ROAD_PATTERN = Pattern.compile('.*[路|大道|巷|胡同|街]\\d+号');
Pattern ROAD_PATTERN = Pattern.compile('.[里,路,道,街,巷,路段,胡同].?\\d+.?\\d+[号,弄]|.[里,路,道,街,巷,路段,胡同].?\\d+.?[号,弄]')
Matcher m = ROAD_PATTERN.matcher(address)
String newAddress
if (m.find()) {
newAddress = m.group()
}
return newAddress
}
def run2MatchV2(fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def region = d['I']
def city = d['J']
if(!d['I'] && !d['J']){
city = d['B']
region = d['C']
}
if(d['D']) {
def name = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
def result = OpenApiUtils.matchDetailDistrict(city, name, region, true)
if (!result && d['E']) {
if (findAddress(d['E'])) {
result = OpenApiUtils.matchDetailDistrict(city, d['E'], region, true)
}
}
if(result){
def matchResult = result.matchResult
if (result.credit > 0.5 && result.matchResult && !result.matchResult.match) {
matchResult = amendMatch(result, d['E'])
}
if (matchResult.match) {
d['AQ'] = result.city
d['AR'] = result.district
d['AS'] = result.name
d['AT'] = result.address
}
}
}
},0)
}
def amendMatch(result, address){
def matchResult = result.matchResult
if(result.credit > 0.65 && matchResult.distance < 2500){
matchResult.match = true
}else if(result.credit>0.5 && address && result.address && findAddress(result.address)){
if(result.address.contains(address) || address.contains(result.address)){
matchResult.match = true
}
}else if(address && address.contains(result.district) && address.contains(result.name)){
matchResult.match = true
}
return matchResult
}
}
@@ -0,0 +1,47 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import net.sf.json.JSONObject
class EvaluationPrice {
static main(args) {
def task = new EvaluationPrice()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值3-V3.xlsx",{ d->
if(d['K'] && d['M']){
def priceMap = getEvaluationPrice(d['J'], d['K'], d['L'], d['D'].toString())
if(priceMap){
d['E'] = priceMap.avgPrice
d['F'] = priceMap.totalPrice
}
}
},0);
}
def getEvaluationPrice(city, region, name, area){
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", region);
params.put("name", name);
params.put("size", area);
params.put("__refresh_cache","1")
JSONObject result = OpenApiClient.evaluate(params);
if (result.getInt("code") == 200) {
if (result.containsKey("result")) {
return result.getJSONObject("result")
}
} else if (result.getInt("code") == 500) {
println("小区估价失败:${params.city} ${params.name} ${params.size} ${result}")
}
}
}
@@ -0,0 +1,629 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.core.RegionData
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.amap.GaoDeMapApiData
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
class TestMatchResult {
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
static main(args) {
def task = new TestMatchResult()
// task.run1("D:\\数据\\重庆农商行\\估值1.xlsx")
task.run2("D:\\数据\\重庆农商行\\估值2-V2.xlsx")
// task.run3("D:\\数据\\重庆农商行\\估值3.xlsx")
// task.nearDistRun("D:\\数据\\重庆农商行\\估值2-V5.xlsx")
// task.run2V1("D:\\数据\\重庆农商行\\估值2-V2.xlsx")
// task.similarScore("D:\\数据\\重庆农商行\\估值2-V5.xlsx")
// task.mapRun("D:\\数据\\重庆农商行\\估值2-V4-map.xlsx");
}
def run1(def fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def region = d['I']
def city = d['J']
if (!d['I'] && !d['J']) {
city = d['B']
region = d['C']
}
def newName = null
if(d['D']=="大川菲尔小城"){
if (d['D']) {
newName = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
}
if (!region || (region && region == d['L'])) {
if (d['D'] && !d['O'] && d['M']) {
if (d['M'] == newName) {
d['O'] = "相同"
}
if (!d['O']) {
def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], d['D'], "district")
if (dbo) {
d['O'] = "别名匹配"
}
}
// if (!d['O'] && d['M'] && d['E']) {
// if (d['E'].toString().contains(d['M'])) {
// d['O'] = "名称in地址"
// }
// }
if (!d['O']) {
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ")
if (py1 == py2) {
d['O'] = "拼音相同"
}
}
if (!d['O'] && d['D'] && d['M'] && newName) {
def word = null
if (newName.toString().length() >= d['M'].toString().length()) {
word = newName.toString().replace(d['M'], "")
} else if (newName.toString().length() < d['M'].toString().length()) {
word = d['M'].toString().replace(newName, "")
}
if (word) {
def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"]
for (def w : words) {
if (word == w) {
d['O'] = "去部分词后相同"
break
}
}
if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) {
if (d['N'] && d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) {
def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName);
if (score >= 0.65) {
println newName + " " + d['M'] + " " + score
d['O'] = "地址相同"
}
}
}
}
}
if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) {
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德")
if (!gaodeRest) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
}
if (gaodeRest && gaodeRest.name) {
def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
if (mapName == d['M'] && gaodeRest.score >= 0.7) {
d['O'] = gaodeRest.source + "匹配"
println newName + " " + d['M'] + " " + gaodeRest.score
}
}
}
}
}
if (d['D'] && !d['O']) {
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德")
def mapName = null
if (gaodeRest && gaodeRest.name) {
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
}
if (!gaodeRest || (gaodeRest && mapName != newName)) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
if (gaodeRest && gaodeRest.name) {
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
}
}
if (gaodeRest && mapName == newName) {
d['P'] = gaodeRest.district
d['Q'] = mapName
d['R'] = gaodeRest.address
d['S'] = gaodeRest.lng
d['T'] = gaodeRest.lat
d['U'] = gaodeRest.type
d['v'] = gaodeRest.source
}
}
}
},0);
}
def similarScore(def fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['D'] && d['M']){
def score = CosineSimilarAlgorithm.getSimilarity(d['D'], d['M']);
d['Q'] = score
}
},0)
}
def run2(def fileName){
ExcelUtils.multiThreadingModify(fileName,{ d->
def region = d['I']
def city = d['J']
if(!d['I']){
city = d['J']
}
// if(d['E']=="巴南区花溪镇花溪新街88号2单元3-4#") {
// println "ss"
if (!region || (region && region == d['L'])) {
if (d['N'] && !d['O'] && d['E']) {
if (d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) {
d['O'] = "地址匹配"
}
}
}
if (!d['O'] && d['M'] && d['E']) {
def matchName = d['M']
if(matchName.toString().endsWith("小区")){
matchName = matchName.toString().replace("小区","")
}
if(matchName.toString().startsWith(d['L'])){
matchName = matchName.toString().replace(d['L'],"")
}
def oldName= NameProcess.nameStandard(city,region, d['E'])
if (oldName.toString().contains(matchName)) {
d['O'] = "名称in地址"
}
}
if (d['E'] && !d['O'] ) {
def gaodeRest = null
if (!gaodeRest && d['E']) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "高德")
}
if(!gaodeRest && d['H']){
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "高德")
}
if (!gaodeRest && d['E']) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "百度")
}
if (!gaodeRest && d['H']) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "百度")
}
if (gaodeRest && gaodeRest.score >= 0.5) {
d['P'] = gaodeRest.district
d['Q'] = gaodeRest.name
d['R'] = gaodeRest.address
d['S'] = gaodeRest.lng
d['T'] = gaodeRest.lat
d['U'] = gaodeRest.type
d['v'] = gaodeRest.source
}
}
if(!d['O']){
def locationMap = getLocation(city, d['E'])
if(locationMap){
def mapRets = findNearestByMap(locationMap.lat, locationMap.lng)
if(mapRets){
def o = mapRets.get(0)
d['W'] = o.cityname
d['X'] = o.adname
d['Y'] = o.name
if(o.address && o.address instanceof String){
d['Z'] = o.address
}
d['AA'] = o.type
d['AB'] = o.distance
d['AC'] = o.location
}
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
if(dbRets){
def b = dbRets.get(0)
d['AD'] = b.region
d['AE'] = b.name
d['AF'] = b.address
d['AG'] = b.distance
}
}
}
// }
},0);
}
def distanceRun(def fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def city = d['J']
if(!d['O'] && d['E'] && d['E'].toString().length()>6){
def locationMap = getLocation(city, d['E'])
if(locationMap){
def mapRets = findNearestByMap(locationMap.lat, locationMap.lng)
if(mapRets){
def o = mapRets.get(0)
d['W'] = o.cityname
d['X'] = o.adname
d['Y'] = o.name
if(o.address && o.address instanceof String){
d['Z'] = o.address
}
d['AA'] = o.type
d['AB'] = o.distance
d['AC'] = o.location
}
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
if(dbRets){
def b = dbRets.get(0)
d['AD'] = b.region
d['AE'] = b.name
d['AF'] = b.address
d['AG'] = b.distance
}
}
}
},0);
}
def nearDistRun(def fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def city = d['B']
// if(!d['I'] && !d['J']){
// city = d['B']
// }
if(d['O'] && d['O']=="周边"){
def locationMap = null
if(d['B'] && d['E']){
locationMap = getLocation(city, d['E'])
}
// if(!locationMap && d['C']){
// locationMap = getLocation(city, d['C'])
// }
if(locationMap){
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
if(dbRets){
def b = dbRets.get(0)
d['K'] = b.city
d['L'] = b.region
d['M'] = b.name
d['N'] = b.address
d['P'] = b.distance
}
}
}
},0);
}
def run2V1(def fileName){
ExcelUtils.multiThreadingModify(fileName,{ d->
def city = d['J']
if(d['O']){
d['AH'] = d['L']
d['AI'] = d['M']
d['AJ'] = d['N']
}
if(!d['O']){
if(d['R'] && d['R'].contains("号")){
def addr = d['R'].toString().substring(0, d['R'].indexOf("号"))
if((addr && d['E'] && d['E'].contains(addr))){
def region = RegionData.formatRegion(city, d['P'])
if(d['U'] && d['U'].toString().contains("住宅")){
d['AH'] = region
d['AI'] = d['Q']
d['AJ'] = d['R']
d['AK'] = d['U']
d['AM'] = d['S']+","+d['T']
d['O'] ="地图匹配"
}else if(!d['U']){
d['AH'] = region
d['AI'] = d['Q']
d['AJ'] = d['R']
d['AK'] = d['U']
d['AM'] = d['S']+","+d['T']
d['O'] ="地图匹配"
}
}
}
}
if(!d['O']){
if(d['Y'] && d['AE']){
def mapName = d['Y'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
if(mapName == d['AE']){
def region = RegionData.formatRegion(city, d['AD'])
d['AH'] = region
d['AI'] = d['AE']
d['AJ'] = d['AF']
d['AK'] = d['DB']
d['O'] ="名称相同"
}else{
if(d['AB'].toString().toDouble()>=d['AG'].toString().toDouble()){
def region = RegionData.formatRegion(city, d['AD'])
d['AH'] = region
d['AI'] = d['AE']
d['AJ'] = d['AF']
d['AK'] = "DB库近"
d['AL'] = d['AG']
d['O'] ="距离比较"
}else{
def region = RegionData.formatRegion(city, d['X'])
d['AH'] = region
d['AI'] = d['Y']
d['AJ'] = d['Z']
d['AK'] = d['AA']
d['AL'] = d['AB']
d['AM'] = d['AC']
d['O'] ="距离比较"
}
}
}else if(d['Y']){
def region = RegionData.formatRegion(city, d['X'])
d['AH'] = region
d['AI'] = d['Y']
d['AJ'] = d['Z']
d['AK'] = d['AA']
d['AL'] = d['AB']
d['AM'] = d['AC']
d['O'] ="采用地图相近"
}else if(d['AE']){
def region = RegionData.formatRegion(city, d['AD'])
d['AH'] = region
d['AI'] = d['AE']
d['AJ'] = d['AF']
d['AK'] = "DB库近"
d['AL'] = d['AG']
d['O'] ="采用DB相近"
}
}
},0);
}
def findNearestByMap(lng, lat){
def rets = GaoDeMapApiData.doSearchByGaoDe("120000|120203|120300|120301|120302|120303|120304", lng, lat, 1000, 1, 10)
return rets
}
public List<DBObject> findNearestDistrict(def city, double lat, double lng, int radiusMeters, int limit) {
DBObject districtFields = new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("address", 1)
districtFields.put("estate_type2", 1)
districtFields.put("lat", 1)
districtFields.put("lng", 1)
DBObject sdq = new BasicDBObject();
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) radiusMeters / 1000.0));
DBObject q = new BasicDBObject();
q.put("cat", "district");
q.put("city", city);
q.put("visible", 0);
q.put('$sd', sdq);
List<DBObject> list = new ArrayList<DBObject>()
Iterator<DBObject> iter = SEAWEED.find(q, districtFields).limit(limit * 2).iterator()
while(iter.hasNext()) {
def o = iter.next()
o.distance = GeoHelper.GetDistance(o.lat/10000000000.0, o.lng/10000000000.0, lat, lng)
list.add(o)
}
if (list.size() == 0) {
return Collections.EMPTY_LIST;
}
list.sort{o1,o2->
return o1.distance <=> o2.distance
}
if (list.size() > limit) {
list = list.subList(0, limit);
}
return list;
}
def getLocation(city, addr){
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(city, addr);
if(reps && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
if(location){
def locMap = location.split(",")
// def lat = dist.lat/10000000000.0;
// def lng = dist.lng/10000000000.0;
// def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
// rMap.distance = distance
// rMap.level = level
rMap.lat = locMap[1].toDouble()
rMap.lng = locMap[0].toDouble()
// rMap.keyword = map.formatted_address
// rMap.location = location
// rMap.region = map.district
return rMap;
}
}
}
return null;
}
def run3(def fileName){
ExcelUtils.multiThreadingModify(fileName,{ d->
def city = d['I']
def region = d['J']
def newName = null
if(d['C']){
if(d['C'].toString().length()<2){
newName = null
}else{
newName = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
}
}
def address = null
if(d['H']){
address = d['H']
}else{
def arrs = d['B'].split("->")
if(arrs && arrs.size()==4){
address = arrs[3]
}else if(arrs && arrs.size()==2){
address = arrs[1]
}
}
if(!region ||(region && d['L'] && region.contains(d['L']))) {
if (d['B'] && !d['O'] && d['M']) {
if (d['M'] == newName) {
d['O'] = "相同"
}
if (!d['O']) {
def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], newName, "district")
if (dbo) {
d['O'] = "别名匹配"
}
}
if (!d['O']) {
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ")
if (py1 == py2) {
d['O'] = "拼音相同"
}
}
if (!d['O'] && d['M'] && newName) {
def word = null
if (newName.toString().length() >= d['M'].toString().length()) {
word = newName.toString().replace(d['M'], "")
} else if (newName.toString().length() < d['M'].toString().length()) {
word = d['M'].toString().replace(newName, "")
}
if (word) {
def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"]
for (def w : words) {
if (word == w) {
d['O'] = "去部分词后相同"
break
}
}
if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) {
if (d['N'] && d['N'].toString().endsWith("号") && address.toString().contains(d['N'])) {
def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName);
if (score >= 0.65) {
println newName + " " + d['M'] + " " + score
d['O'] = "地址相同"
}
}
}
}
}
if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) {
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德")
if (!gaodeRest) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "百度")
}
if (gaodeRest && gaodeRest.name) {
def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
if (mapName == d['M'] && gaodeRest.score >= 0.7) {
d['O'] = gaodeRest.source + "匹配"
println newName + " " + d['M'] + " " + gaodeRest.score
}
}
}
}
}
if(newName && !d['O']){
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德")
def mapName = null
if(gaodeRest && gaodeRest.name){
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
}
if(!gaodeRest || (gaodeRest && mapName != newName)) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
if(gaodeRest && gaodeRest.name){
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
}
}
if(gaodeRest && mapName== newName){
d['P'] = gaodeRest.district
d['Q'] = mapName
d['R'] = gaodeRest.address
d['S'] = gaodeRest.lng
d['T'] = gaodeRest.lat
d['U'] = gaodeRest.type
d['v'] = gaodeRest.source
}
}
},0);
}
def mapRun(fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def city = d['J']
def region = d['I']
if(city && region && d['Q'] && d['O']!="距离比较" && d['O']!="地址匹配"){
def gaodeRest = FangjiaClientUtils.getMapData(city, region, d['Q'], null, "住宅", "高德")
if(gaodeRest){
d['V'] = gaodeRest.city
d['W'] = gaodeRest.district
d['X'] = gaodeRest.name
d['Y'] = gaodeRest.address
d['Z'] = gaodeRest.lng
d['AA'] = gaodeRest.lat
d['AB'] = gaodeRest.type
}
d['AC'] = NameProcess.nameStandard(city,region, d['Q'])
}
},0)
}
}
@@ -0,0 +1,89 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
class TestPrice {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new TestPrice()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值1-V2.xlsx",{ d->
if(d['K'] && d['M']){
def dbo = findThirdPrice(d['K'], d['L'], d['M'], 202012, "lianjia_esf_price")
if(dbo){
d['AU'] = dbo.lianjia_esf_price
}
def district = CommonDatautils.findDistrict(d['K'], d['L'], d['M'], "district")
def startDate = 20201101
def endDate = 20210601
def trades = findTrades(district, startDate, endDate)
if(trades){
d['AV'] = (int)(trades.sum()/trades.size())
d['AW'] = trades.size()
}
}
},0);
}
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject(QueryOperators.GTE, s_date))
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
def obj = null;
for(def d : dbcursor){
obj = d;
break;
}
return obj;
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
// q.put("source", "链家在线")
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def prices = []
for(def d : dbcorsor){
if(d.avg_price){
prices << d.avg_price
}
}
return prices
}
}
@@ -1,39 +1,106 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class FieldStat {
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
static main(args) {
def task = new FieldStat();
task.run()
// task.run()
task.run1()
// task.run2()
}
def run1(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def dist_total = 0
def addr_alias_total = 0
def rets = []
rets << ["城市","别名总量","有别名小区数量","小区总量","有别名小区占比"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias", "addr_alias"]);
def alias_index = 0
def alias_total = 0
for(def d : districts){
def alias = d.alias
if(alias){
alias_index++
alias_total = alias_total + alias.size()
}
}
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
}
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
// println dist_total
// println alias_total
// println addr_alias_total
}
def run(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\½¨ÐÐÊý¾ÝÈںϽá¹ûͳ¼Æ_0425.xlsx", { row ->
def city = row['A']
ExcelUtils.multiThreadingModify("D:\\数据\\城市统计.xlsx", { row ->
def city = row['B']
if(city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
def total = districts.size()
row['D'] = total
def aliasIndex = 0
def aliasTotalCount = 0
for(def d : districts){
def alias = d.alias
if(alias){
aliasIndex++
aliasTotalCount = aliasTotalCount + alias.size()
}
}
// row['J'] = aliasTotalCount
// row['K'] = aliasIndex
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['B'] = p+"%"
// row['M'] = total
println city +" "+p
// def aliasIndex = 0
// def aliasTotalCount = 0
// for(def d : districts){
// def alias = d.alias
// if(alias){
// aliasIndex++
// aliasTotalCount = aliasTotalCount + alias.size()
// }
// }
//// row['J'] = aliasTotalCount
//// row['K'] = aliasIndex
// def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
// row['B'] = p+"%"
//// row['M'] = total
// println city +" "+p
}
}, 1);
}, 0);
println "finish"
}
def run2(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def rets = []
rets << ["城市","建筑年代覆盖率","物业类型覆盖率","建筑类型覆盖率","区域最高小区均价覆盖率","区域最低小区均价覆盖率"]
for(def c : citys){
def obj = findCoverPrecent(c)
if(obj){
def datePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.finishBuildingDate*100)
def estateTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.estateType2*100)
def buildTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.buildType*100)
rets << [c, datePrecent+"%", estateTypePrecent+"%", buildTypePrecent+"%", "100%", "100%"]
}
}
ExcelUtils.write("d:\\数据\\92个城市覆盖率统计.xlsx", rets, "统计")
}
def findCoverPrecent(def city){
def q = new BasicDBObject()
q.put("city", city)
def dbcursor = CITY_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1)
def obj = null
for(def d : dbcursor){
obj = d
}
return obj
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.gj.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
class RegionExport {
static main(args) {
def task = new RegionExport()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("d:\\Êý¾Ý\\Ç×ÁÚ\\Ð¡Çø³ÇÊÐÁбí.xls",{d->
def city = d['B']
def newCity = RegionData.formatCity(city)
if(newCity){
def regions = CommonUtils.getAllRegions(newCity, "region")
def regionStr = ""
regions.each{k,v->
regionStr = regionStr + k+"\n"
}
d['C'] = regionStr
}
},0)
}
}
@@ -24,8 +24,8 @@ class ExportData {
static main(args) {
def task = new ExportData();
// task.exportThird()
task.exportMap();
task.exportThird()
// task.exportMap();
}
def exportMap(){
@@ -71,7 +71,7 @@ class ExportData {
def exportThird(){
def citys = ["╩╢╟╡","ю╔и╫","а╛тф╦ш","до╬╘","дом╗","кчг╗","кужщ","л╘жщ","нчнЩ","пЛжщ","янЁг","яОжщ","уР╫╜"]
def citys = ["ÖØÇì"]
for(def city : citys){
def rets = []
rets << ["城市","区域","名称","别名"]
@@ -3,13 +3,18 @@ package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
JSONObject jsonObj = null
try {
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
if(address){
url = url + "&address="+address
}
// def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
def content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
@@ -44,4 +49,34 @@ class FangjiaClientUtils {
println content
}
public static def searchMapData(def city, def keyword, def types, def mapCode){
def jsonObj = null
try {
def url = "http://192.168.0.19:38080/FangJiaDataService/search?city="+city+"&keyword="+keyword+"&types="+types+"&mapCode="+mapCode+"&limit=10"
def content = new HttpRequest().newGet(url, null, "utf-8")
// def ret = content.replace("[","").replace("]","")
jsonObj = JSONArray.fromObject(content)
} catch (Exception e) {
return null;
}
return jsonObj;
}
public static Map<String, Object> JsonToMap(JSONObject json) {
Map<String, Object> columnValMap = new HashMap<String, Object>();
Set<Object> jsonKeys = json.keySet();
for (Object key : jsonKeys) {
Object JsonValObj = json.get(key);
if (key instanceof JSONObject) {
columnValMap.put((String) key, JsonToMap((JSONObject) JsonValObj));
} else {
columnValMap.put((String) key, JsonValObj);
}
}
return columnValMap;
}
}
@@ -0,0 +1,236 @@
package com.fangjia.script.gj.es
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.lucene.queryparser.classic.QueryParser
class EsMonthHouseStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new EsMonthHouseStat()
// task.run()
task.tradeRun()
// task.houseRun()
}
def run(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def rets = []
def dates = ["2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12","2021-01","2021-02","2021-03","2021-04","2021-05"]
def headers = ["城市"]
headers.addAll(dates)
rets << headers
for(def city : citys){
def datas = []
datas << city
for(def d : dates){
def date = DateUtils.parseFromTightYYYYMMDD(d+"-01")
def startDate = DateUtils.formatDateByPattern(date, "yyyyMMdd")
def endDate = DateUtils.addMonth(date, 1).replace("-","")
def count = findHouseCount(city, startDate.toInteger(), endDate.toInteger())
datas << count
}
rets << datas
println datas
}
ExcelUtils.write("d:\\数据\\建行92个城市房源统计.xlsx", rets, "统计")
}
def tradeRun(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def rets = []
def rets2 = []
//,
//"2021-01","2021-02","2021-03","2021-04","2021-05"
//def dates = ["2020-01","2020-02","2020-03","2020-04","2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12"]
def dates = ["2021-04"]
def headers = ["城市"]
headers.addAll(dates)
rets << headers
def headers2 = ["城市"]
headers2.addAll(dates)
rets2 << headers2
for(def city : citys){
def datas = []
datas << city
def datas2 = []
datas2 << city
for(def d : dates){
def startDate = DateUtils.parseFromTightYYYYMMDD(d+"-01")
def endDateStr = DateUtils.addMonth(startDate, 1).replace("-","")
def endDate = DateUtils.parseFromTightYYYYMMDD2(endDateStr)
//def count = findTrades(city, startDate, endDate)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def oldCount = 0
def newCount = 0
for(def dist : districts){
def countMap = findTradesDetail(dist, startDate, endDate)
oldCount = oldCount + countMap.total_old
newCount = newCount + countMap.total_new
println dist.city + " " + dist.name +" "+countMap.total_old+" "+countMap.total_new
}
datas << oldCount
datas2 << newCount
println city +" "+ oldCount +" "+newCount
}
rets << datas
rets2 << datas2
println datas
}
ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重前匹配).xlsx", rets, "统计")
ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重后匹配).xlsx", rets2, "统计")
}
def houseRun(){
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
def city = d['B']
if(city == "湖州"){
def count = findHouseCount(city, 20210301, 20210601)
d['E'] = count
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
if(distCount){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
}
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['D'] = precent+"%"
}
},0)
}
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
def aliasStr = null;
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def findHouseCount(def city, def startDate, def endDate){
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
def total = 0
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
return total
}
def findTrades(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
// q.put("region", dist.region)
// q.put("source", "链家在线")
// def alias = []
// if(dist.alias){
// if(!dist.alias.contains(dist.name)){
// dist.alias << dist.name
// }
// alias = dist.alias
// }else{
// alias << dist.name
// }
// if(alias){
// q.put("district_name", new BasicDBObject('$in', alias))
// }
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def count = DEAL_PRICE_COLL.count(q)
return count
}
def findTradesDetail(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
}
}
def rMap = [:]
rMap.total_old = total
rMap.total_new = rets.size()
return rMap
}
}
@@ -0,0 +1,66 @@
package com.fangjia.script.gj.es
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
class EsSelectTest {
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
static main(args) {
def task = new EsSelectTest()
task.run()
}
def run(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("request", "/ws/czbank"))
//.must(QueryBuilders.termQuery("request_body", "findSample"))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
if(houses) {
for (int k = 0; k < houses.size(); k++) {
println k
}
}
def endDate = new Date()
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
dataSource.setProperties(connectionProperties)
StringBuffer sqlBuffer = new StringBuffer();
String d = DateUtil.formatByPattern(new Date(), "yyyy.MM.dd")
long startD = DateUtil.getBeforeMinute(new Date(), -7).getTime();
long endD = DateUtil.getBeforeMinute(endDate, 1).getTime();
println(DateUtil.formatByPattern(new Date(startD), "yyyy-MM-dd HH:mm:ss")+"===="+DateUtil.formatByPattern(new Date(endD), "yyyy-MM-dd HH:mm:ss"));
sqlBuffer.append("select request_time FROM \"fangjia-log*\" WHERE 1=1");
sqlBuffer.append(" and request='/ws/czbank' and request_body='findSample'")
ResultSet rs = null
try {
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
rs = statement.executeQuery(sqlBuffer.toString())
} catch (Exception e) {
e.printStackTrace()
}
StringBuffer msgBuffer = new StringBuffer();
while (rs.next()) {
def responsetime = rs.getLong("responsetime");
def timestamp = DateUtil.formatByPattern(new Date(rs.getLong("time")), "yyyy-MM-dd HH:mm")
println responsetime
}
}
}
@@ -0,0 +1,171 @@
package com.fangjia.script.gj.location
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.text.SimpleDateFormat
import java.util.regex.Matcher
import java.util.regex.Pattern;
class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) {
def task = new LocationTest()
// task.run()
task.run2()
}
def run(){
def regionPointsMap = findRegionRange()
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
def index = 0
def rets = []
for(def dist : districts){
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
if(!flag){
index++
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
def loactionMap = matchMap(dist)
if(loactionMap){
ret << "地图"
ret << loactionMap.region
ret << loactionMap.mapName
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
if(dist.region==loactionMap.region){
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
}else{
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
}
}else{
ret << ""
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
}
rets << ret
}
}
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内")
}
def run2(){
ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx")
}
def matchMap(def dist){
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
def locationMap = null
if(gaodeRest){
locationMap = getMapLocation(dist, gaodeRest);
if(locationMap){
return locationMap
}
}
def baiduRest = null
if (!locationMap) {
baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
}
if(baiduRest){
locationMap = getMapLocation(dist, baiduRest);
}
return locationMap
}
def getMapLocation(def dist, def rets){
def lat = null
def lng = null
def region = null
def mapName = null
for(def r : rets){
if(!r.city){
continue
}
if(!r.city.contains(dist.city)){
continue
}
def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
if(dist.name == name){
lat = r.lat
lng = r.lng
mapName = r.name
region = r.district
}else{
def word = null
if (name.length() >= dist.name.length()) {
word = name.replace(dist.name, "")
} else if (name.length() < dist.name.length()) {
word = dist.name.replace(name, "")
}
if (word) {
def words = ["商住楼", "小区","公寓","别墅"]
for (def w : words) {
if (word == w) {
lat = r.lat
lng = r.lng
region = r.district
mapName = r.name
break
}
}
if(!lng){
Pattern pattern = Pattern.compile("\\d+号楼");
Matcher matcher = pattern.matcher(word);
if(matcher.find()){
lat = r.lat
lng = r.lng
region = r.district
mapName = r.name
break
}
}
}
}
if(lng){
break
}
}
if(lng){
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName]
}
return null
}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
//Point point = new Point(lng, lat);
//def flag = InsidePolygon.aboutZone(point, pointList)
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
def findRegionRange(){
def q = new BasicDBObject()
q.put("city","福州市")
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion("福州",d.region)
regionMap.put(region, d.polyline)
}
return regionMap
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
class CrawlDistrictIdSelect {
@@ -14,8 +15,8 @@ class CrawlDistrictIdSelect {
static main(args) {
def task = new CrawlDistrictIdSelect()
task.run()
// task.synch()
// task.run()
task.synch()
}
def synch(){
@@ -50,13 +51,13 @@ class CrawlDistrictIdSelect {
def findCrawlData(){
def q = new BasicDBObject()
q.put("city","¸£ÖÝ")
q.put("source", "°²¾Ó¿Í")
q.put("s_date",new BasicDBObject('$gte', 202001))
def dbcursor = thirdPartyColl.find(q)
// q.put("city","¸£ÖÝ")
q.put("source", "Öî¸ðÕÒ·¿")
q.put("s_date",202010)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = "/community/view/"
def keyList =[]
def prefix = "xiaoqu.zhuge.com/"
def keyList =new HashSet()
for(def db : dbcursor){
def url = null
if(!db.url){
@@ -67,12 +68,11 @@ class CrawlDistrictIdSelect {
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
def region = RegionData.formatRegion(db.city, db.region)
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com"))
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
@@ -83,15 +83,17 @@ class CrawlDistrictIdSelect {
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "anjuke.com"
map.root = "zhuge.com"
map.city_short= shortCity
map.city = db.city
map.region = region
map.region = db.region
map.district_id = district_id
map.source_url = url
ids << map
keyList << key
}else{
println "ÖØ¸´£º"+key
}
}else{
println url +" Á´½Ó²»ºÏÒªÇó"
@@ -32,6 +32,22 @@ class InsidePolygon {
return list;
}
public static List<Point> createPoint2(String points){
String[] array = points.split(";");
List<Point> list = new ArrayList<Point>();
for(int i =0 ;i<array.length-1;i++){
def pointArrs = array[i].split("[,|\\|]")
Point point = new Point();
double x = Double.parseDouble(pointArrs[0]);
double y = Double.parseDouble(pointArrs[1]);
point.setLon(x);
point.setLat(y);
list.add(point);
}
return list;
}
public static List<Map<String, Object>> getPoints(String points){
String[] array = points.split("-");
@@ -167,4 +183,44 @@ class InsidePolygon {
return false;
return true;
}
public static boolean isPtInPoly (double ALon , double ALat , List<Point> ps) {
int iSum, iCount, iIndex;
double dLon1 = 0, dLon2 = 0, dLat1 = 0, dLat2 = 0, dLon;
if (ps.size() < 3) {
return false;
}
iSum = 0;
iCount = ps.size();
for (iIndex = 0; iIndex<iCount;iIndex++) {
if (iIndex == iCount - 1) {
dLon1 = ps[iIndex].getLon()
dLat1 = ps[iIndex].getLat()
dLon2 = ps[0].getLon()
dLat2 = ps[0].getLat()
} else {
dLon1 = ps[iIndex].getLon()
dLat1 = ps[iIndex].getLat()
dLon2 = ps[iIndex + 1].getLon()
dLat2 = ps[iIndex + 1].getLat()
}
// 以下语句判断A点是否在边的两端点的水平平行线之间,在则可能有交点,开始判断交点是否在左射线上
if (((ALat >= dLat1) && (ALat < dLat2)) || ((ALat >= dLat2) && (ALat < dLat1))) {
if (Math.abs(dLat1 - dLat2) > 0) {
//得到 A点向左射线与边的交点的x坐标:
dLon = dLon1 - ((dLon1 - dLon2) * (dLat1 - ALat) ) / (dLat1 - dLat2);
// 如果交点在A点左侧(说明是做射线与 边的交点),则射线与边的全部交点数加一:
if (dLon < ALon) {
iSum++;
}
}
}
}
if ((iSum % 2) != 0) {
return true;
}
return false;
}
}
@@ -1,6 +1,9 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
@@ -21,7 +24,8 @@ import java.util.concurrent.Executors
* @author shenjian
* @since 2021/5/24
*/
class SeaweedAroundPoi {
@BatchOwner(Watcher.ShenJian)
class SeaweedAroundPoi extends BaseBatch {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
@@ -170,4 +174,12 @@ class SeaweedAroundPoi {
put("ÂÃÓξ°µã","110000");
put("ͼÊé¹Ý","140500");
}};
@Override
void process(Object args) {
def types = listAllType()
//dealSpecialSeaweedAround(types)
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
}
}
@@ -139,12 +139,17 @@ class GaoDeUtils {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
//throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
// 多线程情况下还是exit有效果,抛异常不会停止程序
if (keyUsed.intValue() >= GAO_DE_AK.size()) {
System.exit(1)
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
break;
}
keyUsed.incrementAndGet()
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
@@ -25,6 +25,7 @@ class CrawalDBUtil {
static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process")
static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics")
static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process")
static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
public static getSeedsCol() {
return seedsColl;
@@ -67,6 +68,10 @@ class CrawalDBUtil {
return map_district_col;
}
public static getDistrictDetailColl(){
return districtDetailColl;
}
static def updateSeedsCount(def id, def count) {
DBObject uquery = new BasicDBObject()
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
def static final SHORT_DATE = Utils.getLastMonth()
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static {
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
@@ -0,0 +1,106 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.script.util.CommonUtils
import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash
class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋"]
//"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋"
static main(args) {
def task = new SynchDistrictPrice()
task.run()
}
def run(){
def citys = ["福州"]//CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
def rets = findCrawlDistrictDetails(city, k, v)
synchData(city, rets, v)
println city +" "+rets.size()
}
}
}
def findCrawlDistrictDetails(city, root, source){
def q = new BasicDBObject()
q.put("city", city)
q.put("root", root)
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("block", 1)
keys.put("name", 1)
keys.put("address", 1)
keys.put("avg_price", 1)
keys.put("root", 1)
keys.put("url", 1)
def rets = []
def dbcursor = districtDetailColl.find(q, keys)
def names = new HashSet()
for(def d : dbcursor){
def s = d.city +"#"+d.region+"#"+d.name
if(!names.contains(s)){
names << s
d.put("district_id", d.url)
d.put("source", source)
d.put("s_date", SHORT_DATE)
d.removeField("root")
d.removeField("url")
d.removeField("_id")
rets << d
}
}
return rets
}
def synchData(def city, def rets, def source){
def q = new BasicDBObject()
q.put("source", source)
q.put("s_date", SHORT_DATE)
q.put("city", city)
def count = thirdPartyColl.count(q)
def addIndex = 0
if(count==0){
try{
thirdPartyColl.insert(rets)
}catch(Exception e){
println e.getMessage()
}
}else{
for(def r : rets){
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(!flag){
addIndex++
thirdPartyColl.save(r)
}
}
}
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
}
def findIsExists(def city, def region, def name, def sDate, def source){
def q = new BasicDBObject()
q.put("source", source)
q.put("s_date", sDate)
q.put("city", city)
q.put("region", region)
q.put("name", name)
def count = thirdPartyColl.count(q)
if(count){
return true
}
return false
}
}
@@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf"
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf"
]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
@@ -309,6 +309,31 @@ public class OpenApiClient {
return null
}
public static JSONObject addressStandard(String city, String address) {
if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) {
throw new IllegalArgumentException();
}
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content)
if (result.getInt("code") == 200) {
return result.getJSONObject("result");
} else if (result.getInt("code") == 500) {
println("${city} ${address} ${result}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) {
try {
Map<String, String> params = new HashMap<String, String>();
@@ -1214,4 +1239,6 @@ public class OpenApiClient {
return null;
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.ProvinceData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -11,45 +12,71 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
def count = 0
List<List<String>> data = Collections.synchronizedList([])
data << ["ID","城市","区域","小区名","经度","纬度","街道"]
CommonUtils.findAllCity().each {def city->
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
if(StringUtils.isBlank(item.township) && item.lng){
count++
def lng = item.lng/10000000000d
def lat = item.lat/10000000000d
def location = "${lng},${lat}"
JSONObject resp = GaoDeApiUtils.geocode(location)
def row = []
row << item._id.toString()
row << item.city
row << item.region
row << item.name
row << lng
row << lat
BasicDBObject dbo = new BasicDBObject()
dbo.id = item._id.toString()
dbo.city = item.city
dbo.region = item.region
dbo.name = item.name
dbo.lng = lng
dbo.lat = lat
if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
row << resp.regeocode.addressComponent.township
dbo.township = resp.regeocode.addressComponent.township
}
println(row)
data << row
SEAWEED_TOWNSHIP.insert(dbo)
}
})
}
ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
//def count = 0
//List<List<String>> data = Collections.synchronizedList([])
//data << ["ID","城市","区域","小区名","经度","纬度","街道"]
//
//CommonUtils.findAllCity().each {def city->
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
// if(StringUtils.isBlank(item.township) && item.lng){
// count++
// def lng = item.lng/10000000000d
// def lat = item.lat/10000000000d
// def location = "${lng},${lat}"
// JSONObject resp = GaoDeApiUtils.geocode(location)
// def row = []
// row << item._id.toString()
// row << item.city
// row << item.region
// row << item.name
// row << lng
// row << lat
// BasicDBObject dbo = new BasicDBObject()
// dbo.id = item._id.toString()
// dbo.city = item.city
// dbo.region = item.region
// dbo.name = item.name
// dbo.lng = lng
// dbo.lat = lat
// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
// row << resp.regeocode.addressComponent.township
// dbo.township = resp.regeocode.addressComponent.township
// }
// println(row)
// data << row
// SEAWEED_TOWNSHIP.insert(dbo)
// }
// })
//}
//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item ->
// if(item["G"] && item["H"] == null){
// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"])))
// }
// println(item)
//})
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item->
// def location = "${item["A"]},${item["B"]}"
// JSONObject resp = GaoDeApiUtils.geocode(location)
// if(resp.regeocode && resp.regeocode.formatted_address){
// item['G'] = resp.regeocode.formatted_address
// println(item)
// }
//})
//def count = 0
//CommonUtils.findAllCity().each {def city->
// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
// if(item.township){
// count++
// }
// }, new BasicDBObject("township", 1))
//}
//println(count)
ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{
println(it)
})
@@ -9,6 +9,11 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
def strt(String addr){
}
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
def city = item["A"]
def address1 = item["B"]
@@ -223,6 +223,32 @@ class GaoDeMapApiData {
doSearchByGaoDe(type, lng, lat, radius, page)
}
}
def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("types", type)
params.put("location", Double.toString(lng)+ ","+ Double.toString(lat))
params.put("radius",radius)
params.put("offset",pageSize)
params.put("page",page)
String content
try {
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
}
JSONObject jsonObj = JSONObject.fromObject(content)
if(jsonObj.get("infocode").equals("10000")){
JSONArray pois = jsonObj.get("pois")
return pois
} else {
println jsonObj.get("infocode")+"exit........"
doSearchByGaoDe(type, lng, lat, radius, page, pageSize)
}
return null
}
private synchronized static fzData(JSONArray pois) {
pois.each { poi ->