@@ -2,13 +2,19 @@ package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.CsvUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
@@ -18,35 +24,44 @@ def count5 = 0
|
||||
def count6 = 0
|
||||
def count7 = 0
|
||||
def count8 = 0
|
||||
def total=4943
|
||||
def count9 = 0
|
||||
def total = datas.size()
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]
|
||||
def regionCount = 0
|
||||
|
||||
def rets = []
|
||||
rets << ["网站有对比价数量","有对比价楼盘量","有对比价占比"]
|
||||
for(def d : datas){
|
||||
def list = []
|
||||
if(d['G']){
|
||||
list << d['G']
|
||||
if(regions.contains(d[3])){
|
||||
regionCount ++
|
||||
}
|
||||
if(d['H']){
|
||||
list << d['H']
|
||||
if(d[6]){
|
||||
list << d[6]
|
||||
}
|
||||
if(d['I']){
|
||||
list << d['I']
|
||||
if(d[7]){
|
||||
list << d[7]
|
||||
}
|
||||
if(d['J']){
|
||||
list << d['J']
|
||||
if(d[8]){
|
||||
list << d[8]
|
||||
}
|
||||
if(d['K']){
|
||||
list << d['K']
|
||||
if(d[9]){
|
||||
list << d[9]
|
||||
}
|
||||
if(d['L']){
|
||||
list << d['L']
|
||||
if(d[10]){
|
||||
list << d[10]
|
||||
}
|
||||
if(d['M']){
|
||||
list << d['M']
|
||||
if(d[11]){
|
||||
list << d[11]
|
||||
}
|
||||
if(d['N']){
|
||||
list << d['N']
|
||||
if(d[12]){
|
||||
list << d[12]
|
||||
}
|
||||
if(d['O']){
|
||||
list << d['O']
|
||||
if(d[13]){
|
||||
list << d[13]
|
||||
}
|
||||
if(d[14]){
|
||||
list << d[14]
|
||||
}
|
||||
if(list.size()==0){
|
||||
count0++
|
||||
@@ -76,18 +91,53 @@ for(def d : datas){
|
||||
if(list.size()==8){
|
||||
count8++
|
||||
}
|
||||
if(list.size()==9){
|
||||
count9++
|
||||
}
|
||||
}
|
||||
|
||||
println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total)))
|
||||
println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total)))
|
||||
println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total)))
|
||||
println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total)))
|
||||
println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total)))
|
||||
println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total)))
|
||||
println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total)))
|
||||
println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total)))
|
||||
println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total)))
|
||||
println("总量:"+ total)
|
||||
def pre1 = DataHandleUtils.getDoubleRound(2, count1*100.0/total)
|
||||
println("count1:"+ pre1)
|
||||
def pre2 = DataHandleUtils.getDoubleRound(2, count2*100.0/total)
|
||||
println("count2:"+ pre2)
|
||||
def pre3 = DataHandleUtils.getDoubleRound(2, count3*100.0/total)
|
||||
println("count3:"+ pre3)
|
||||
|
||||
def pre4 = DataHandleUtils.getDoubleRound(2, count4*100.0/total)
|
||||
println("count4:"+ pre4)
|
||||
|
||||
def pre5 = DataHandleUtils.getDoubleRound(2, count5*100.0/total)
|
||||
println("count5:"+ pre5)
|
||||
|
||||
def pre6 = DataHandleUtils.getDoubleRound(2, count6*100.0/total)
|
||||
println("count6:"+ pre6)
|
||||
|
||||
def pre7 = DataHandleUtils.getDoubleRound(2, count7*100.0/total)
|
||||
println("count7:"+ pre7)
|
||||
|
||||
def pre8 = DataHandleUtils.getDoubleRound(2, count8*100.0/total)
|
||||
println("count8:"+ pre8)
|
||||
|
||||
def pre9 = DataHandleUtils.getDoubleRound(2, count9*100.0/total)
|
||||
println("count9:"+ pre9)
|
||||
|
||||
rets << ["1家",count1, pre1+"%"]
|
||||
rets << ["2家",count2, pre2+"%"]
|
||||
rets << ["3家",count3, pre3+"%"]
|
||||
rets << ["4家",count4, pre4+"%"]
|
||||
rets << ["5家",count5, pre5+"%"]
|
||||
rets << ["6家",count6, pre6+"%"]
|
||||
rets << ["7家",count7, pre7+"%"]
|
||||
rets << ["8家",count8, pre8+"%"]
|
||||
rets << ["9家",count9, pre9+"%"]
|
||||
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
|
||||
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
|
||||
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
|
||||
|
||||
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
|
||||
|
||||
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
|
||||
@@ -128,6 +178,18 @@ println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total)))
|
||||
//}
|
||||
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split("@\\|@",-1)
|
||||
rows << row
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -65,12 +65,12 @@ class LandDetailCrawl {
|
||||
}
|
||||
|
||||
def updateData(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\土地\\土地成交3月(2).xlsx",0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\土地\\土地-21-06.xlsx",0, 1)
|
||||
if(datas && datas.size()>0){
|
||||
for(def data : datas){
|
||||
def id = data['A']
|
||||
DBObject upMap = new BasicDBObject()
|
||||
def area = data['E']
|
||||
def area = data['I']
|
||||
|
||||
if(area){
|
||||
if(area){
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
class WuXiAddressTest {
|
||||
static main(args) {
|
||||
def task = new WuXiAddressTest()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
// def datas = ExcelUtils.read("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",0,1)
|
||||
// for(def d : datas){
|
||||
// def locationStr = getLocationInfo(d['D'], d['Q'])
|
||||
// def township = getStreetInfo(locationStr)
|
||||
// def name = getJuWeiHuiInfo(locationStr);
|
||||
//
|
||||
// }
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d->
|
||||
if(!d['S']){
|
||||
def locationStr = getLocationInfo(d['D'], d['Q'])
|
||||
def township = getStreetInfo(locationStr)
|
||||
def name = getJuWeiHuiInfo(locationStr, "村委会", "1000");
|
||||
if(!name){
|
||||
name = getJuWeiHuiInfo(locationStr,"村委会", "2000");
|
||||
if(name){
|
||||
d['T'] = 2000
|
||||
}
|
||||
}else{
|
||||
d['T'] = 1000
|
||||
}
|
||||
|
||||
if(!name){
|
||||
name = getJuWeiHuiInfo(locationStr, "村委会", "3000");
|
||||
if(name){
|
||||
d['T'] = 3000
|
||||
}
|
||||
}
|
||||
d['R'] = township
|
||||
d['S'] = name
|
||||
}
|
||||
|
||||
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def getLocationInfo(def city, def address){
|
||||
def gloJson = GaoDeApiUtils.geo(city, address)
|
||||
if(gloJson) {
|
||||
def glocation = null
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
glocation = geocodes.get(0).location
|
||||
}
|
||||
return glocation
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def getStreetInfo(def glocation){
|
||||
def township = null
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
return township;
|
||||
}
|
||||
|
||||
def getJuWeiHuiInfo(def glocation, def keyword, def radius){
|
||||
def json = GaoDeApiUtils.around2(glocation, radius,keyword, null)
|
||||
def name = null
|
||||
if(json && json.info=="OK" && json.pois){
|
||||
def pois = json.pois
|
||||
if(pois){
|
||||
def poi = pois[0]
|
||||
name = poi.name
|
||||
}
|
||||
}
|
||||
return name;
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-04-01"
|
||||
static def statDate = "2021-05-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+16
@@ -65,6 +65,22 @@ class FangjiaClientUtils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def addressStand(def address){
|
||||
def jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> JsonToMap(JSONObject json) {
|
||||
Map<String, Object> columnValMap = new HashMap<String, Object>();
|
||||
Set<Object> jsonKeys = json.keySet();
|
||||
|
||||
+1146
File diff suppressed because it is too large
Load Diff
+36
-1
@@ -18,8 +18,10 @@ class EsMonthHouseStat {
|
||||
static main(args) {
|
||||
def task = new EsMonthHouseStat()
|
||||
// task.run()
|
||||
task.tradeRun()
|
||||
// task.tradeRun()
|
||||
// task.houseRun()
|
||||
|
||||
task.statHouseUpdateCount()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -233,4 +235,37 @@ class EsMonthHouseStat {
|
||||
rMap.total_new = rets.size()
|
||||
return rMap
|
||||
}
|
||||
|
||||
|
||||
def statHouseUpdateCount(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def root = "lianjia.com"
|
||||
def rets = []
|
||||
rets << ["城市", "5月上旬","5月中旬","5月下旬", "6月前10天"]
|
||||
for(def city : citys){
|
||||
def count1 = findHouseCount(city, root, 20210501, 20210510)
|
||||
def count2 = findHouseCount(city, root, 20210510, 20210520)
|
||||
def count3 = findHouseCount(city, root, 20210520, 20210530)
|
||||
def count4 = findHouseCount(city, root, 20210601, 20210610)
|
||||
rets << [city, count1, count2, count3, count4]
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\房源分析\\城市房源分析.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
def findHouseCount(def city, def root, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='${root}'"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
}
|
||||
|
||||
+229
-20
@@ -1,11 +1,14 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -21,9 +24,16 @@ class LocationTest {
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new LocationTest()
|
||||
Date date=new Date(1622802609);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
String str = format.format(date);
|
||||
System.out.println(str);
|
||||
|
||||
// def task = new LocationTest()
|
||||
// task.run()
|
||||
task.run2()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.run4()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -43,11 +53,14 @@ class LocationTest {
|
||||
|
||||
def loactionMap = matchMap(dist)
|
||||
if(loactionMap){
|
||||
ret << "地图"
|
||||
ret << loactionMap.source
|
||||
ret << loactionMap.region
|
||||
ret << loactionMap.mapName
|
||||
ret << loactionMap.address
|
||||
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
ret << distance
|
||||
ret << loactionMap.lng
|
||||
ret << loactionMap.lat
|
||||
if(dist.region==loactionMap.region){
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
|
||||
}else{
|
||||
@@ -61,31 +74,188 @@ class LocationTest {
|
||||
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内")
|
||||
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx")
|
||||
def datas = ExcelUtils.read("d:\\数据\\福州\\福州小区.xlsx", 0,1)
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称", "来源","地图区域", "地图名称","距离", "地图地址", "地图经度", "地图纬度"]
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
def count = d['H'].toInteger()
|
||||
def label = d['G'].toInteger()
|
||||
if(count && count>=4 && label!=-1){
|
||||
index++
|
||||
def city = d['C']
|
||||
def region = d['D']
|
||||
def name = d['B']
|
||||
|
||||
def dist = CommonDatautils.queryDistrictById(d['A'].toString())
|
||||
def ret = []
|
||||
ret << city
|
||||
ret << region
|
||||
ret << name
|
||||
|
||||
def loactionMap = matchMap(dist)
|
||||
if(loactionMap){
|
||||
ret << loactionMap.source
|
||||
ret << loactionMap.region
|
||||
ret << loactionMap.mapName
|
||||
def distance = null
|
||||
if(loactionMap.source=="高德"){
|
||||
distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
}else if(loactionMap.source=="百度"){
|
||||
distance = GeoHelper.GetDistance(dist.lat2/10000000000.0, dist.lng2/10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
}
|
||||
ret << distance
|
||||
ret << loactionMap.address
|
||||
ret << loactionMap.lng
|
||||
ret << loactionMap.lat
|
||||
if(dist.region==loactionMap.region){
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
|
||||
}else{
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
|
||||
}
|
||||
}else{
|
||||
// ret << ""
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
|
||||
}
|
||||
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\福州\\相同经纬度小区4.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d->
|
||||
def distance = null
|
||||
if(d['G']){
|
||||
distance = d['G'].toDouble()
|
||||
}
|
||||
if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){
|
||||
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
|
||||
if(mapName.contains("-") && mapName.endsWith("楼")){
|
||||
mapName = mapName.substring(0, mapName.indexOf("-"))
|
||||
}
|
||||
if(d['C']==mapName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region", d['B'])
|
||||
q.put("name", d['C'])
|
||||
q.put("cat", 'district')
|
||||
q.put("status", 0)
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
def lng = null
|
||||
def lat = null
|
||||
if(d['D']=="高德"){
|
||||
lng = (d['I'].toDouble()*10000000000).longValue()
|
||||
lat = (d['J'].toDouble()*10000000000).longValue()
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble())
|
||||
if(gmap){
|
||||
lng2 = (gmap.get("lng")*10000000000).longValue()
|
||||
lat2 = (gmap.get("lat")*10000000000).longValue()
|
||||
}
|
||||
|
||||
}else{
|
||||
lng2 = (d['I'].toDouble()*10000000000).longValue()
|
||||
lat2 = (d['J'].toDouble()*10000000000).longValue()
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble())
|
||||
if(gmap){
|
||||
lng = (gmap.get("lng")*10000000000).longValue()
|
||||
lat = (gmap.get("lat")*10000000000).longValue()
|
||||
}
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("lng", lng)
|
||||
udata.put("lat", lat)
|
||||
udata.put("lng2", lng2)
|
||||
udata.put("lat2", lat2)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
|
||||
|
||||
def dbo = DbUtils.seaweed.findOne(q)
|
||||
d['K'] = "修正"
|
||||
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
|
||||
def run4(){
|
||||
ExcelUtils.modify("d:\\数据\\福州\\福州相同经纬度小区4.xlsx", {d->
|
||||
def region = d['B']
|
||||
def name = d['C']
|
||||
if(!d['K']){
|
||||
def dist = ["city":d['A'], "region":region, "name":name]
|
||||
def loactionMap = matchMap(dist)
|
||||
if(loactionMap){
|
||||
d['D'] << loactionMap.source
|
||||
d['E'] << loactionMap.region
|
||||
d['F'] << loactionMap.mapName
|
||||
def distance = null
|
||||
if(loactionMap.source=="高德"){
|
||||
distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
}else if(loactionMap.source=="百度"){
|
||||
distance = GeoHelper.GetDistance(dist.lat2/10000000000.0, dist.lng2/10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
}
|
||||
d['G'] << distance
|
||||
d['H'] << loactionMap.address
|
||||
d['I'] << loactionMap.lng
|
||||
d['J'] << loactionMap.lat
|
||||
if(dist.region==loactionMap.region){
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
|
||||
}else{
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
|
||||
}
|
||||
}else{
|
||||
// ret << ""
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def matchMap(def dist){
|
||||
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
|
||||
def locationMap = null
|
||||
def gdlocationMap = null
|
||||
def bdlocationMap = null
|
||||
if(gaodeRest){
|
||||
locationMap = getMapLocation(dist, gaodeRest);
|
||||
if(locationMap){
|
||||
return locationMap
|
||||
gdlocationMap = getMapLocation(dist, gaodeRest);
|
||||
if(gdlocationMap){
|
||||
gdlocationMap.source = "高德"
|
||||
}
|
||||
}
|
||||
|
||||
def baiduRest = null
|
||||
if (!locationMap) {
|
||||
baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
}
|
||||
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
if(baiduRest){
|
||||
locationMap = getMapLocation(dist, baiduRest);
|
||||
bdlocationMap = getMapLocation(dist, baiduRest);
|
||||
if(bdlocationMap){
|
||||
bdlocationMap.source = "百度"
|
||||
}
|
||||
}
|
||||
return locationMap
|
||||
|
||||
if(gdlocationMap && bdlocationMap){
|
||||
if(gdlocationMap.score>=bdlocationMap.score){
|
||||
return gdlocationMap
|
||||
}else{
|
||||
return bdlocationMap
|
||||
}
|
||||
}else if(gdlocationMap){
|
||||
return gdlocationMap
|
||||
}else if(bdlocationMap){
|
||||
return bdlocationMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getMapLocation(def dist, def rets){
|
||||
@@ -93,6 +263,8 @@ class LocationTest {
|
||||
def lng = null
|
||||
def region = null
|
||||
def mapName = null
|
||||
def address = null
|
||||
def score = 0
|
||||
for(def r : rets){
|
||||
if(!r.city){
|
||||
continue
|
||||
@@ -100,12 +272,20 @@ class LocationTest {
|
||||
if(!r.city.contains(dist.city)){
|
||||
continue
|
||||
}
|
||||
def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
|
||||
def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("建设中","").replace("钢混","").trim()
|
||||
if(dist.name == name){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
mapName = r.name
|
||||
region = r.district
|
||||
address = r.address
|
||||
|
||||
if(r.district.contains(dist.region)){
|
||||
score = 10
|
||||
}else {
|
||||
score = 5
|
||||
}
|
||||
|
||||
}else{
|
||||
def word = null
|
||||
if (name.length() >= dist.name.length()) {
|
||||
@@ -116,11 +296,18 @@ class LocationTest {
|
||||
if (word) {
|
||||
def words = ["商住楼", "小区","公寓","别墅"]
|
||||
for (def w : words) {
|
||||
if (word == w) {
|
||||
if (word == w ) {
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
address = r.address
|
||||
|
||||
if(r.district.contains(dist.region)){
|
||||
score = 9
|
||||
}else {
|
||||
score = 5
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -132,6 +319,30 @@ class LocationTest {
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
address = r.address
|
||||
if(r.district.contains(dist.region)){
|
||||
score = 8
|
||||
}else {
|
||||
score = 5
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if(!lng){
|
||||
Pattern pattern = Pattern.compile("[A-Za-z]区");
|
||||
Matcher matcher = pattern.matcher(word);
|
||||
if(matcher.find()){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
address = r.address
|
||||
if(r.district.contains(dist.region)){
|
||||
score = 7
|
||||
}else {
|
||||
score = 5
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -142,10 +353,8 @@ class LocationTest {
|
||||
}
|
||||
}
|
||||
if(lng){
|
||||
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName]
|
||||
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score]
|
||||
}
|
||||
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
+301
@@ -0,0 +1,301 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AddressStandard {
|
||||
static main(args) {
|
||||
def task = new AddressStandard()
|
||||
task.run()//地址标准化
|
||||
task.test()//统计标准化后的结果
|
||||
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.multiThreadingModify("d://数据//地址标准化//阿里接口拆分数据(1).xlsx",{d->
|
||||
def json = FangjiaClientUtils.addressStand(d['A'])
|
||||
|
||||
if(json && json.status!=0){
|
||||
println d['A'] +" "+json
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def run(){
|
||||
// def datas = ExcelUtils.read("d://数据//地址标准化//阿里接口拆分数据(1-test).xlsx",0,1)
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("d://数据//地址标准化//阿里接口拆分数据(1).xlsx",{d->
|
||||
if(d['A']){
|
||||
for(def i=18;i<d.size();i++){
|
||||
d[i] = null
|
||||
}
|
||||
|
||||
def json = FangjiaClientUtils.addressStand(d['A'])
|
||||
if(json && json.entities){
|
||||
def entities = json.entities
|
||||
for(def m : entities){
|
||||
if(m.type=="prov"){
|
||||
d['S'] = m.word
|
||||
}
|
||||
if(m.type=="city"){
|
||||
d['T'] = m.word
|
||||
}
|
||||
if(m.type=="dist") {
|
||||
d['U'] = m.word
|
||||
}
|
||||
if(m.type=="town") {
|
||||
d['V'] = m.word
|
||||
}
|
||||
if(m.type=="community") {
|
||||
d['W'] = m.word
|
||||
}
|
||||
if(m.type=="road") {
|
||||
d['X'] = m.word
|
||||
}
|
||||
|
||||
if(m.type=="roadno") {
|
||||
d['Y'] = m.word
|
||||
}
|
||||
if(m.type=="dir") {
|
||||
d['Z'] = m.word
|
||||
}
|
||||
if(m.type=="poi") {
|
||||
d['AA'] = m.word
|
||||
}
|
||||
if(m.type=="subpoi") {
|
||||
d['AB'] = m.word
|
||||
}
|
||||
if(m.type=="group") {
|
||||
d['AC'] = m.word
|
||||
}
|
||||
if(m.type=="houseno") {
|
||||
d['AD'] = m.word
|
||||
}
|
||||
if(m.type=="cellno") {
|
||||
d['AE'] = m.word
|
||||
}
|
||||
if(m.type=="floorno") {
|
||||
d['AF'] = m.word
|
||||
}
|
||||
if(m.type=="roomno") {
|
||||
d['AG'] = m.word
|
||||
}
|
||||
if(m.type=="dev"){
|
||||
d['AH'] = m.word
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
|
||||
def test(){
|
||||
def AtomicInteger provCount = new AtomicInteger(0)
|
||||
def AtomicInteger cityCount = new AtomicInteger(0)
|
||||
def AtomicInteger distCount = new AtomicInteger(0)
|
||||
def AtomicInteger townCount = new AtomicInteger(0)
|
||||
def AtomicInteger roadCount = new AtomicInteger(0)
|
||||
def AtomicInteger roadnoConnt = new AtomicInteger(0)
|
||||
def AtomicInteger communityCount = new AtomicInteger(0)
|
||||
def AtomicInteger poiCount = new AtomicInteger(0)
|
||||
def AtomicInteger subpoiCount = new AtomicInteger(0)
|
||||
def AtomicInteger housenoCount = new AtomicInteger(0)
|
||||
def AtomicInteger cellnoCount = new AtomicInteger(0)
|
||||
def AtomicInteger floornoCount = new AtomicInteger(0)
|
||||
def AtomicInteger roomnoCount = new AtomicInteger(0)
|
||||
def AtomicInteger devCount = new AtomicInteger(0)
|
||||
|
||||
def AtomicInteger total = new AtomicInteger(0)
|
||||
|
||||
ExcelUtils.multiThreadingModify("d://数据//地址标准化//阿里接口拆分数据(1).xlsx",{d->
|
||||
total.addAndGet(1)
|
||||
def sameCount = 0
|
||||
def list = []
|
||||
def a_prov = d['B']?d['B']:""
|
||||
if(a_prov && (a_prov.toString().endsWith("省") || a_prov.toString().endsWith("自治区"))){
|
||||
a_prov = a_prov.replace("维吾尔自治区","").replace("回族自治区","").replace("壮族自治区","").replace("自治区省","").replace("自治区","").replace("省","")
|
||||
}
|
||||
def b_prov = d['S']?d['S']:""
|
||||
if(b_prov && (b_prov.toString().endsWith("省") || b_prov.toString().endsWith("自治区"))){
|
||||
b_prov = b_prov.replace("维吾尔自治区","").replace("回族自治区","").replace("壮族自治区","").replace("自治区省","").replace("自治区","").replace("省","")
|
||||
}
|
||||
if(a_prov == b_prov){
|
||||
provCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "prov"
|
||||
}
|
||||
|
||||
def a_city = d['C']?d['C']:""
|
||||
def b_city = d['T']?d['T']:""
|
||||
if(a_city && a_city.toString().endsWith("市")){
|
||||
a_city = a_city.replace("市","")
|
||||
}
|
||||
if(b_city && b_city.toString().endsWith("市")){
|
||||
b_city = b_city.replace("市","")
|
||||
}
|
||||
if(a_city == b_city){
|
||||
cityCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "city"
|
||||
}
|
||||
|
||||
def a_dist = d['D']?d['D']:""
|
||||
def b_dist = d['U']?d['U']:""
|
||||
if(a_dist && (a_dist.endsWith("区")||a_dist.endsWith("县"))){
|
||||
a_dist = a_dist.replace("新区","").replace("区","").replace("县","")
|
||||
}
|
||||
if(b_dist && (b_dist.endsWith("区")||b_dist.endsWith("县"))){
|
||||
b_dist = b_dist.replace("新区","").replace("区","").replace("县","")
|
||||
}
|
||||
|
||||
if(a_dist == b_dist){
|
||||
distCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "dist"
|
||||
}
|
||||
|
||||
def a_town = d['E']?d['E']:""
|
||||
def b_town = d['V']?d['V']:""
|
||||
if(a_town==b_town){
|
||||
townCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "town"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def a_road = d['F']?d['F']:""
|
||||
def b_road = d['X']?d['X']:""
|
||||
if(a_road == b_road){
|
||||
roadCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "road"
|
||||
}
|
||||
|
||||
|
||||
def a_roadno = d['H']?d['H']:""
|
||||
def b_roadno = d['Y']?d['Y']:""
|
||||
if(a_roadno == b_roadno){
|
||||
roadnoConnt.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "roadno"
|
||||
}
|
||||
|
||||
def a_community = d['I']?d['I']:""
|
||||
def b_community = d['W']?d['W']:""
|
||||
if(a_community == b_community){
|
||||
communityCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "community"
|
||||
}
|
||||
|
||||
def a_poi = d['J']?d['J']:""
|
||||
def b_poi = d['AA']?d['AA']:""
|
||||
if(a_poi == b_poi){
|
||||
poiCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "poi"
|
||||
}
|
||||
|
||||
def a_subpoi = d['K']?d['K']:""
|
||||
def b_subpoi = d['AB']?d['AB']:""
|
||||
if(a_subpoi == b_subpoi){
|
||||
subpoiCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "subpoi"
|
||||
}
|
||||
|
||||
def a_houseno = d['L']?d['L'].replace("号楼","").replace("幢楼","").replace("栋号","").replace("幢","").replace("栋","").replace("座","").replace("号","").replace("楼","").replace("第","").replace("楼",""):""
|
||||
def b_houseno = d['AD']?d['AD'].replace("号楼","").replace("幢楼","").replace("栋号","").replace("幢","").replace("栋","").replace("座","").replace("号","").replace("楼","").replace("第","").replace("楼",""):""
|
||||
if(a_houseno == b_houseno){
|
||||
housenoCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "houseno"
|
||||
}
|
||||
|
||||
def a_cellno = d['M']?chNumberConver(d['M'].replace("单元","")):""
|
||||
def b_cellno = d['AE']?chNumberConver(d['AE'].replace("单元","")):""
|
||||
if(a_cellno == b_cellno){
|
||||
cellnoCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "cellno"
|
||||
}
|
||||
|
||||
def a_floorno = d['N']?d['N'].replace("层楼","").replace("层","").replace("楼",""):""
|
||||
def b_floorno = d['AF']?d['AF'].replace("层楼","").replace("层","").replace("楼",""):""
|
||||
if(a_floorno == b_floorno){
|
||||
floornoCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "floorno"
|
||||
}
|
||||
|
||||
def a_roomno = d['O']?d['O'].replace("室号","").replace("号门","").replace("室","").replace("房","").replace("户","").replace("号","").replace("门",""):""
|
||||
def b_roomno = d['AG']?d['AG'].replace("室号","").replace("号门","").replace("室","").replace("房","").replace("户","").replace("号","").replace("门",""):""
|
||||
if(a_roomno == b_roomno){
|
||||
roomnoCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "roomno"
|
||||
}
|
||||
|
||||
def a_dev = d['Q']?d['Q']:''
|
||||
def b_dev = d['AH']?d['AH']:''
|
||||
if(a_dev == b_dev){
|
||||
devCount.addAndGet(1)
|
||||
sameCount++
|
||||
}else{
|
||||
list << "dev"
|
||||
}
|
||||
|
||||
d['AI'] = sameCount
|
||||
d['AJ'] = list.join(",")
|
||||
|
||||
},0)
|
||||
|
||||
def totalCount = total.get()
|
||||
println totalCount
|
||||
def prov_pre = DataHandleUtils.getDoubleRound(2, provCount.get()*100.0/totalCount)
|
||||
def city_pre = DataHandleUtils.getDoubleRound(2, cityCount.get()*100.0/totalCount)
|
||||
def dist_pre = DataHandleUtils.getDoubleRound(2, distCount.get()*100.0/totalCount)
|
||||
def town_pre = DataHandleUtils.getDoubleRound(2, townCount.get()*100.0/totalCount)
|
||||
def community_pre = DataHandleUtils.getDoubleRound(2, communityCount.get()*100.0/totalCount)
|
||||
def road_pre = DataHandleUtils.getDoubleRound(2, roadCount.get()*100.0/totalCount)
|
||||
def roadno_pre = DataHandleUtils.getDoubleRound(2, roadnoConnt.get()*100.0/totalCount)
|
||||
def poi_pre = DataHandleUtils.getDoubleRound(2, poiCount.get()*100.0/totalCount)
|
||||
def subpoi_pre = DataHandleUtils.getDoubleRound(2, subpoiCount.get()*100.0/totalCount)
|
||||
def houseno_pre = DataHandleUtils.getDoubleRound(2, housenoCount.get()*100.0/totalCount)
|
||||
def cellno_pre = DataHandleUtils.getDoubleRound(2, cellnoCount.get()*100.0/totalCount)
|
||||
def floor_pre = DataHandleUtils.getDoubleRound(2, floornoCount.get()*100.0/totalCount)
|
||||
def roomno_pre = DataHandleUtils.getDoubleRound(2, roomnoCount.get()*100.0/totalCount)
|
||||
def datas = []
|
||||
datas << ["总量", "prov","city","dist","town","community","road","roadno","poi","subpoi","houseno","cellno","floor","roomno"]
|
||||
datas << [totalCount,prov_pre,city_pre,dist_pre,town_pre,community_pre,road_pre,roadno_pre,poi_pre,subpoi_pre,houseno_pre,cellno_pre,floor_pre,roomno_pre]
|
||||
ExcelUtils.write("d://数据//地址标准化//测试结果1.xlsx", datas, "数据")
|
||||
|
||||
}
|
||||
|
||||
def chNumberConver(def s){
|
||||
if(s && s.size()==1){
|
||||
return s.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
|
||||
}
|
||||
return s;
|
||||
}
|
||||
}
|
||||
+18
-9
@@ -15,8 +15,8 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
static main(args) {
|
||||
def task = new CrawlDistrictIdSelect()
|
||||
// task.run()
|
||||
task.synch()
|
||||
task.run()
|
||||
// task.synch()
|
||||
}
|
||||
|
||||
def synch(){
|
||||
@@ -38,25 +38,34 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
def run(){
|
||||
def datas = findCrawlData()
|
||||
def list = []
|
||||
datas.each {k,v->
|
||||
if(v){
|
||||
for(def d : v){
|
||||
d.c_date = new Date()
|
||||
d.m_date = new Date()
|
||||
districtseedsColl.save(d)
|
||||
list << d
|
||||
}
|
||||
}
|
||||
|
||||
if(list.size()==10000){
|
||||
districtseedsColl.insert(list)
|
||||
list = []
|
||||
}
|
||||
}
|
||||
if(list.size()>0){
|
||||
districtseedsColl.insert(list)
|
||||
}
|
||||
}
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city","福州")
|
||||
q.put("source", "诸葛找房")
|
||||
q.put("s_date",202010)
|
||||
q.put("source", "安居客")
|
||||
q.put("s_date",202012)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = "xiaoqu.zhuge.com/"
|
||||
def prefix = "anjuke.com/"
|
||||
def keyList =new HashSet()
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
@@ -68,11 +77,11 @@ class CrawlDistrictIdSelect {
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
|
||||
def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com"))
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com"))
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
@@ -83,7 +92,7 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "zhuge.com"
|
||||
map.root = "anjuke.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
|
||||
+5
-4
@@ -166,8 +166,8 @@ class Esf58Crawl extends BaseBatch{
|
||||
def addr = JQueryUtils.selectText(".desc", detailDocument);
|
||||
if(info){
|
||||
if(info.size()>=4){
|
||||
city = info[2].replace("小区","")
|
||||
region = info[3].replace("小区","")
|
||||
city = info[2].replace("小区","").trim()
|
||||
region = info[3].replace("小区","").trim()
|
||||
}
|
||||
}
|
||||
if(!addr){
|
||||
@@ -212,8 +212,8 @@ class Esf58Crawl extends BaseBatch{
|
||||
def saveThirdParty(metadata){
|
||||
DBObject obj = new BasicDBObject();
|
||||
obj.put("district_id", metadata.url);
|
||||
obj.put("city", metadata.city);
|
||||
obj.put("region", metadata.region);
|
||||
obj.put("city", metadata.city.trim());
|
||||
obj.put("region", metadata.region.trim());
|
||||
obj.put("block", metadata.block);
|
||||
obj.put("name", metadata.name);
|
||||
obj.put("s_date", SHORT_DATE);
|
||||
@@ -221,6 +221,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
obj.put("addres", metadata.address);
|
||||
obj.put("avg_price", metadata.avg_price.toInteger());
|
||||
obj.put("c_date", new Date());
|
||||
println obj.toString()
|
||||
thirdPartyColl.save(obj);
|
||||
}
|
||||
|
||||
|
||||
+12
-7
@@ -4,14 +4,17 @@ import com.fangjia.script.util.CommonUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.solr.common.util.Hash
|
||||
|
||||
/**
|
||||
* 福州农商行
|
||||
* 将抓取的小区均价同步到第三方均价表中
|
||||
*/
|
||||
class SynchDistrictPrice {
|
||||
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
def sourceMap = ["jiwu.com":"吉屋"]
|
||||
//"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋"
|
||||
|
||||
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
"anjuke.com":"安居客"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -19,13 +22,14 @@ class SynchDistrictPrice {
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = ["福州"]//CommonUtils.findAllCity()
|
||||
def citys = CommonUtils.findAllCity()
|
||||
|
||||
sourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
def rets = findCrawlDistrictDetails(city, k, v)
|
||||
synchData(city, rets, v)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, v)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,7 +80,7 @@ class SynchDistrictPrice {
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
|
||||
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
|
||||
}else{
|
||||
for(def r : rets){
|
||||
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
@@ -85,9 +89,10 @@ class SynchDistrictPrice {
|
||||
thirdPartyColl.save(r)
|
||||
}
|
||||
}
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
|
||||
}
|
||||
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
|
||||
|
||||
}
|
||||
|
||||
def findIsExists(def city, def region, def name, def sDate, def source){
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202102);
|
||||
def count2 = findTotalCount(source, 202103);
|
||||
def count1 = findTotalCount(source, 202105);
|
||||
def count2 = findTotalCount(source, 202106);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import com.fangjia.script.lixue.util.DateFormat
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.hssf.record.BoolErrRecord
|
||||
|
||||
@@ -389,8 +390,8 @@ public class ExcelUtils{
|
||||
try{
|
||||
closure.call(row)
|
||||
int count = counter.incrementAndGet()
|
||||
if(count % 100 == 0){
|
||||
println("执行进度:${com.fangjia.core.util.DataUtil.roundDouble((count * 100)/total, 2)}%")
|
||||
if(count % 1000 == 0){
|
||||
println(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+"执行进度:${com.fangjia.core.util.DataUtil.roundDouble((count * 100)/total, 2)}%")
|
||||
}
|
||||
}catch (Exception e){
|
||||
println(row)
|
||||
|
||||
+2
-2
@@ -121,9 +121,9 @@ public class OpenApiClient {
|
||||
params.put("suggestName", suggestName)
|
||||
}
|
||||
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/matchV2", params, "utf-8")
|
||||
while (content == null) {
|
||||
content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8")
|
||||
content = new HttpRequest().get("http:///open.fangjia.com/address/matchV2", params, "utf-8")
|
||||
println("${city} ${input} ${cat}程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
|
||||
+35
@@ -173,6 +173,41 @@ class GaoDeApiUtils {
|
||||
}
|
||||
|
||||
|
||||
def static around2(String location,String radius, String keywords = null, types = null) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("key", GAO_DE_AK)
|
||||
if(keywords){
|
||||
params.put("keywords", keywords)
|
||||
}
|
||||
if(types){
|
||||
params.put("types", types)
|
||||
}
|
||||
if(location){
|
||||
params.put("location", location)
|
||||
}
|
||||
if(radius){
|
||||
params.put("radius", radius)
|
||||
}
|
||||
params.put("sortrule", "distance")
|
||||
String content = null;
|
||||
try {
|
||||
content = new HttpRequest().newGet("https://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||
if(jsonObj && jsonObj.get("infocode").equals("10000")){
|
||||
return jsonObj
|
||||
}else if(jsonObj && jsonObj.get("infocode").equals("20003")){
|
||||
return null;
|
||||
} else {
|
||||
return around(location, radius, keywords, types)
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def static searchKey(def keyword, def cityCode){
|
||||
def url = "https://www.amap.com/service/poiInfo?query_type=TQUERY&pagesize=20&pagenum=1&qii=true&cluster_state=5&need_utd=true&utd_sceneid=1000&div=PC1000&addr_poi_merge=true&is_classify=true&zoom=10&city="+cityCode+"&geoobj=113.082026%7C22.516777%7C115.059608%7C22.922141&keywords="+URLEncoder.encode(keyword,"UTF-8")
|
||||
String content
|
||||
|
||||
Reference in New Issue
Block a user