This commit is contained in:
2021-07-05 16:41:05 +08:00
parent 7e89258b9e
commit 72a3f78abe
15 changed files with 1986 additions and 77 deletions
@@ -2,13 +2,19 @@ package com.fangjia.script.fuzhou
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CsvUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -18,35 +24,44 @@ def count5 = 0
def count6 = 0
def count7 = 0
def count8 = 0
def total=4943
def count9 = 0
def total = datas.size()
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]
def regionCount = 0
def rets = []
rets << ["网站有对比价数量","有对比价楼盘量","有对比价占比"]
for(def d : datas){
def list = []
if(d['G']){
list << d['G']
if(regions.contains(d[3])){
regionCount ++
}
if(d['H']){
list << d['H']
if(d[6]){
list << d[6]
}
if(d['I']){
list << d['I']
if(d[7]){
list << d[7]
}
if(d['J']){
list << d['J']
if(d[8]){
list << d[8]
}
if(d['K']){
list << d['K']
if(d[9]){
list << d[9]
}
if(d['L']){
list << d['L']
if(d[10]){
list << d[10]
}
if(d['M']){
list << d['M']
if(d[11]){
list << d[11]
}
if(d['N']){
list << d['N']
if(d[12]){
list << d[12]
}
if(d['O']){
list << d['O']
if(d[13]){
list << d[13]
}
if(d[14]){
list << d[14]
}
if(list.size()==0){
count0++
@@ -76,18 +91,53 @@ for(def d : datas){
if(list.size()==8){
count8++
}
if(list.size()==9){
count9++
}
}
println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total)))
println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total)))
println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total)))
println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total)))
println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total)))
println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total)))
println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total)))
println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total)))
println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total)))
println("总量:"+ total)
def pre1 = DataHandleUtils.getDoubleRound(2, count1*100.0/total)
println("count1:"+ pre1)
def pre2 = DataHandleUtils.getDoubleRound(2, count2*100.0/total)
println("count2:"+ pre2)
def pre3 = DataHandleUtils.getDoubleRound(2, count3*100.0/total)
println("count3:"+ pre3)
def pre4 = DataHandleUtils.getDoubleRound(2, count4*100.0/total)
println("count4:"+ pre4)
def pre5 = DataHandleUtils.getDoubleRound(2, count5*100.0/total)
println("count5:"+ pre5)
def pre6 = DataHandleUtils.getDoubleRound(2, count6*100.0/total)
println("count6:"+ pre6)
def pre7 = DataHandleUtils.getDoubleRound(2, count7*100.0/total)
println("count7:"+ pre7)
def pre8 = DataHandleUtils.getDoubleRound(2, count8*100.0/total)
println("count8:"+ pre8)
def pre9 = DataHandleUtils.getDoubleRound(2, count9*100.0/total)
println("count9:"+ pre9)
rets << ["1家",count1, pre1+"%"]
rets << ["2家",count2, pre2+"%"]
rets << ["3家",count3, pre3+"%"]
rets << ["4家",count4, pre4+"%"]
rets << ["5家",count5, pre5+"%"]
rets << ["6家",count6, pre6+"%"]
rets << ["7家",count7, pre7+"%"]
rets << ["8家",count8, pre8+"%"]
rets << ["9家",count9, pre9+"%"]
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM")
ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计")
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
DingDingMessageUtil.sendTextMessage(access_token, s)
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
@@ -128,6 +178,18 @@ println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total)))
//}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("@\\|@",-1)
rows << row
}
return rows;
}
@@ -65,12 +65,12 @@ class LandDetailCrawl {
}
def updateData(){
def datas = ExcelUtils.read("D:\\数据\\土地\\土地成交3月(2).xlsx",0, 1)
def datas = ExcelUtils.read("D:\\数据\\土地\\土地-21-06.xlsx",0, 1)
if(datas && datas.size()>0){
for(def data : datas){
def id = data['A']
DBObject upMap = new BasicDBObject()
def area = data['E']
def area = data['I']
if(area){
if(area){
@@ -0,0 +1,90 @@
package com.fangjia.script.gj
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
class WuXiAddressTest {
static main(args) {
def task = new WuXiAddressTest()
task.run()
}
def run(){
// def datas = ExcelUtils.read("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",0,1)
// for(def d : datas){
// def locationStr = getLocationInfo(d['D'], d['Q'])
// def township = getStreetInfo(locationStr)
// def name = getJuWeiHuiInfo(locationStr);
//
// }
ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d->
if(!d['S']){
def locationStr = getLocationInfo(d['D'], d['Q'])
def township = getStreetInfo(locationStr)
def name = getJuWeiHuiInfo(locationStr, "村委会", "1000");
if(!name){
name = getJuWeiHuiInfo(locationStr,"村委会", "2000");
if(name){
d['T'] = 2000
}
}else{
d['T'] = 1000
}
if(!name){
name = getJuWeiHuiInfo(locationStr, "村委会", "3000");
if(name){
d['T'] = 3000
}
}
d['R'] = township
d['S'] = name
}
},0)
}
def getLocationInfo(def city, def address){
def gloJson = GaoDeApiUtils.geo(city, address)
if(gloJson) {
def glocation = null
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
glocation = geocodes.get(0).location
}
return glocation
}
return null
}
def getStreetInfo(def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
}
}
}
return township;
}
def getJuWeiHuiInfo(def glocation, def keyword, def radius){
def json = GaoDeApiUtils.around2(glocation, radius,keyword, null)
def name = null
if(json && json.info=="OK" && json.pois){
def pois = json.pois
if(pois){
def poi = pois[0]
name = poi.name
}
}
return name;
}
}
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-04-01"
static def statDate = "2021-05-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -65,6 +65,22 @@ class FangjiaClientUtils {
}
public static def addressStand(def address){
def jsonObj = null
try {
def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
def content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
e.printStackTrace();
}
return jsonObj;
}
public static Map<String, Object> JsonToMap(JSONObject json) {
Map<String, Object> columnValMap = new HashMap<String, Object>();
Set<Object> jsonKeys = json.keySet();
@@ -18,8 +18,10 @@ class EsMonthHouseStat {
static main(args) {
def task = new EsMonthHouseStat()
// task.run()
task.tradeRun()
// task.tradeRun()
// task.houseRun()
task.statHouseUpdateCount()
}
def run(){
@@ -233,4 +235,37 @@ class EsMonthHouseStat {
rMap.total_new = rets.size()
return rMap
}
def statHouseUpdateCount(){
def citys = CommonDatautils.findAllCity()
def root = "lianjia.com"
def rets = []
rets << ["城市", "5月上旬","5月中旬","5月下旬", "6月前10天"]
for(def city : citys){
def count1 = findHouseCount(city, root, 20210501, 20210510)
def count2 = findHouseCount(city, root, 20210510, 20210520)
def count3 = findHouseCount(city, root, 20210520, 20210530)
def count4 = findHouseCount(city, root, 20210601, 20210610)
rets << [city, count1, count2, count3, count4]
}
ExcelUtils.write("d:\\数据\\房源分析\\城市房源分析.xlsx", rets, "数据")
}
def findHouseCount(def city, def root, def startDate, def endDate){
def aliasStr = null;
def total = 0
try{
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='${root}'"
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
}
@@ -1,11 +1,14 @@
package com.fangjia.script.gj.location
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
@@ -21,9 +24,16 @@ class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) {
def task = new LocationTest()
Date date=new Date(1622802609);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String str = format.format(date);
System.out.println(str);
// def task = new LocationTest()
// task.run()
task.run2()
// task.run2()
// task.run3()
// task.run4()
}
def run(){
@@ -43,11 +53,14 @@ class LocationTest {
def loactionMap = matchMap(dist)
if(loactionMap){
ret << "地图"
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if(dist.region==loactionMap.region){
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
}else{
@@ -61,31 +74,188 @@ class LocationTest {
}
}
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内")
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内")
}
def run2(){
ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx")
def datas = ExcelUtils.read("d:\\数据\\福州\\福州小区.xlsx", 0,1)
def rets = []
rets << ["城市", "区域", "名称", "来源","地图区域", "地图名称","距离", "地图地址", "地图经度", "地图纬度"]
def index = 0
for(def d : datas){
def count = d['H'].toInteger()
def label = d['G'].toInteger()
if(count && count>=4 && label!=-1){
index++
def city = d['C']
def region = d['D']
def name = d['B']
def dist = CommonDatautils.queryDistrictById(d['A'].toString())
def ret = []
ret << city
ret << region
ret << name
def loactionMap = matchMap(dist)
if(loactionMap){
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
def distance = null
if(loactionMap.source=="高德"){
distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
}else if(loactionMap.source=="百度"){
distance = GeoHelper.GetDistance(dist.lat2/10000000000.0, dist.lng2/10000000000.0, loactionMap.lat, loactionMap.lng)
}
ret << distance
ret << loactionMap.address
ret << loactionMap.lng
ret << loactionMap.lat
if(dist.region==loactionMap.region){
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
}else{
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
}
}else{
// ret << ""
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
}
rets << ret
}
}
ExcelUtils.write("d:\\数据\\福州\\相同经纬度小区4.xlsx", rets, "数据")
}
def run3(){
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d->
def distance = null
if(d['G']){
distance = d['G'].toDouble()
}
if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
if(mapName.contains("-") && mapName.endsWith("楼")){
mapName = mapName.substring(0, mapName.indexOf("-"))
}
if(d['C']==mapName){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['B'])
q.put("name", d['C'])
q.put("cat", 'district')
q.put("status", 0)
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(d['D']=="高德"){
lng = (d['I'].toDouble()*10000000000).longValue()
lat = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng2 = (gmap.get("lng")*10000000000).longValue()
lat2 = (gmap.get("lat")*10000000000).longValue()
}
}else{
lng2 = (d['I'].toDouble()*10000000000).longValue()
lat2 = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng = (gmap.get("lng")*10000000000).longValue()
lat = (gmap.get("lat")*10000000000).longValue()
}
}
def udata = new BasicDBObject()
udata.put("lng", lng)
udata.put("lat", lat)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
def dbo = DbUtils.seaweed.findOne(q)
d['K'] = "修正"
}
}
},0)
}
def run4(){
ExcelUtils.modify("d:\\数据\\福州\\福州相同经纬度小区4.xlsx", {d->
def region = d['B']
def name = d['C']
if(!d['K']){
def dist = ["city":d['A'], "region":region, "name":name]
def loactionMap = matchMap(dist)
if(loactionMap){
d['D'] << loactionMap.source
d['E'] << loactionMap.region
d['F'] << loactionMap.mapName
def distance = null
if(loactionMap.source=="高德"){
distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
}else if(loactionMap.source=="百度"){
distance = GeoHelper.GetDistance(dist.lat2/10000000000.0, dist.lng2/10000000000.0, loactionMap.lat, loactionMap.lng)
}
d['G'] << distance
d['H'] << loactionMap.address
d['I'] << loactionMap.lng
d['J'] << loactionMap.lat
if(dist.region==loactionMap.region){
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
}else{
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
}
}else{
// ret << ""
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
}
}
},0)
}
def matchMap(def dist){
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
def locationMap = null
def gdlocationMap = null
def bdlocationMap = null
if(gaodeRest){
locationMap = getMapLocation(dist, gaodeRest);
if(locationMap){
return locationMap
gdlocationMap = getMapLocation(dist, gaodeRest);
if(gdlocationMap){
gdlocationMap.source = "高德"
}
}
def baiduRest = null
if (!locationMap) {
baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
}
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
if(baiduRest){
locationMap = getMapLocation(dist, baiduRest);
bdlocationMap = getMapLocation(dist, baiduRest);
if(bdlocationMap){
bdlocationMap.source = "百度"
}
}
return locationMap
if(gdlocationMap && bdlocationMap){
if(gdlocationMap.score>=bdlocationMap.score){
return gdlocationMap
}else{
return bdlocationMap
}
}else if(gdlocationMap){
return gdlocationMap
}else if(bdlocationMap){
return bdlocationMap
}
return null
}
def getMapLocation(def dist, def rets){
@@ -93,6 +263,8 @@ class LocationTest {
def lng = null
def region = null
def mapName = null
def address = null
def score = 0
for(def r : rets){
if(!r.city){
continue
@@ -100,12 +272,20 @@ class LocationTest {
if(!r.city.contains(dist.city)){
continue
}
def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("建设中","").replace("钢混","").trim()
if(dist.name == name){
lat = r.lat
lng = r.lng
mapName = r.name
region = r.district
address = r.address
if(r.district.contains(dist.region)){
score = 10
}else {
score = 5
}
}else{
def word = null
if (name.length() >= dist.name.length()) {
@@ -116,11 +296,18 @@ class LocationTest {
if (word) {
def words = ["商住楼", "小区","公寓","别墅"]
for (def w : words) {
if (word == w) {
if (word == w ) {
lat = r.lat
lng = r.lng
region = r.district
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
score = 9
}else {
score = 5
}
break
}
}
@@ -132,6 +319,30 @@ class LocationTest {
lng = r.lng
region = r.district
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
score = 8
}else {
score = 5
}
break
}
}
if(!lng){
Pattern pattern = Pattern.compile("[A-Za-z]区");
Matcher matcher = pattern.matcher(word);
if(matcher.find()){
lat = r.lat
lng = r.lng
region = r.district
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
score = 7
}else {
score = 5
}
break
}
}
@@ -142,10 +353,8 @@ class LocationTest {
}
}
if(lng){
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName]
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score]
}
return null
}
@@ -0,0 +1,301 @@
package com.fangjia.script.gj.test
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.atomic.AtomicInteger
class AddressStandard {
static main(args) {
def task = new AddressStandard()
task.run()//地址标准化
task.test()//统计标准化后的结果
}
def run2(){
ExcelUtils.multiThreadingModify("d://数据//地址标准化//阿里接口拆分数据(1).xlsx",{d->
def json = FangjiaClientUtils.addressStand(d['A'])
if(json && json.status!=0){
println d['A'] +" "+json
}
},0)
}
def run(){
// def datas = ExcelUtils.read("d://数据//地址标准化//阿里接口拆分数据(1-test).xlsx",0,1)
ExcelUtils.multiThreadingModify("d://数据//地址标准化//阿里接口拆分数据(1).xlsx",{d->
if(d['A']){
for(def i=18;i<d.size();i++){
d[i] = null
}
def json = FangjiaClientUtils.addressStand(d['A'])
if(json && json.entities){
def entities = json.entities
for(def m : entities){
if(m.type=="prov"){
d['S'] = m.word
}
if(m.type=="city"){
d['T'] = m.word
}
if(m.type=="dist") {
d['U'] = m.word
}
if(m.type=="town") {
d['V'] = m.word
}
if(m.type=="community") {
d['W'] = m.word
}
if(m.type=="road") {
d['X'] = m.word
}
if(m.type=="roadno") {
d['Y'] = m.word
}
if(m.type=="dir") {
d['Z'] = m.word
}
if(m.type=="poi") {
d['AA'] = m.word
}
if(m.type=="subpoi") {
d['AB'] = m.word
}
if(m.type=="group") {
d['AC'] = m.word
}
if(m.type=="houseno") {
d['AD'] = m.word
}
if(m.type=="cellno") {
d['AE'] = m.word
}
if(m.type=="floorno") {
d['AF'] = m.word
}
if(m.type=="roomno") {
d['AG'] = m.word
}
if(m.type=="dev"){
d['AH'] = m.word
}
}
}
}
},0)
}
def test(){
def AtomicInteger provCount = new AtomicInteger(0)
def AtomicInteger cityCount = new AtomicInteger(0)
def AtomicInteger distCount = new AtomicInteger(0)
def AtomicInteger townCount = new AtomicInteger(0)
def AtomicInteger roadCount = new AtomicInteger(0)
def AtomicInteger roadnoConnt = new AtomicInteger(0)
def AtomicInteger communityCount = new AtomicInteger(0)
def AtomicInteger poiCount = new AtomicInteger(0)
def AtomicInteger subpoiCount = new AtomicInteger(0)
def AtomicInteger housenoCount = new AtomicInteger(0)
def AtomicInteger cellnoCount = new AtomicInteger(0)
def AtomicInteger floornoCount = new AtomicInteger(0)
def AtomicInteger roomnoCount = new AtomicInteger(0)
def AtomicInteger devCount = new AtomicInteger(0)
def AtomicInteger total = new AtomicInteger(0)
ExcelUtils.multiThreadingModify("d://数据//地址标准化//阿里接口拆分数据(1).xlsx",{d->
total.addAndGet(1)
def sameCount = 0
def list = []
def a_prov = d['B']?d['B']:""
if(a_prov && (a_prov.toString().endsWith("省") || a_prov.toString().endsWith("自治区"))){
a_prov = a_prov.replace("维吾尔自治区","").replace("回族自治区","").replace("壮族自治区","").replace("自治区省","").replace("自治区","").replace("省","")
}
def b_prov = d['S']?d['S']:""
if(b_prov && (b_prov.toString().endsWith("省") || b_prov.toString().endsWith("自治区"))){
b_prov = b_prov.replace("维吾尔自治区","").replace("回族自治区","").replace("壮族自治区","").replace("自治区省","").replace("自治区","").replace("省","")
}
if(a_prov == b_prov){
provCount.addAndGet(1)
sameCount++
}else{
list << "prov"
}
def a_city = d['C']?d['C']:""
def b_city = d['T']?d['T']:""
if(a_city && a_city.toString().endsWith("市")){
a_city = a_city.replace("市","")
}
if(b_city && b_city.toString().endsWith("市")){
b_city = b_city.replace("市","")
}
if(a_city == b_city){
cityCount.addAndGet(1)
sameCount++
}else{
list << "city"
}
def a_dist = d['D']?d['D']:""
def b_dist = d['U']?d['U']:""
if(a_dist && (a_dist.endsWith("区")||a_dist.endsWith("县"))){
a_dist = a_dist.replace("新区","").replace("区","").replace("县","")
}
if(b_dist && (b_dist.endsWith("区")||b_dist.endsWith("县"))){
b_dist = b_dist.replace("新区","").replace("区","").replace("县","")
}
if(a_dist == b_dist){
distCount.addAndGet(1)
sameCount++
}else{
list << "dist"
}
def a_town = d['E']?d['E']:""
def b_town = d['V']?d['V']:""
if(a_town==b_town){
townCount.addAndGet(1)
sameCount++
}else{
list << "town"
}
def a_road = d['F']?d['F']:""
def b_road = d['X']?d['X']:""
if(a_road == b_road){
roadCount.addAndGet(1)
sameCount++
}else{
list << "road"
}
def a_roadno = d['H']?d['H']:""
def b_roadno = d['Y']?d['Y']:""
if(a_roadno == b_roadno){
roadnoConnt.addAndGet(1)
sameCount++
}else{
list << "roadno"
}
def a_community = d['I']?d['I']:""
def b_community = d['W']?d['W']:""
if(a_community == b_community){
communityCount.addAndGet(1)
sameCount++
}else{
list << "community"
}
def a_poi = d['J']?d['J']:""
def b_poi = d['AA']?d['AA']:""
if(a_poi == b_poi){
poiCount.addAndGet(1)
sameCount++
}else{
list << "poi"
}
def a_subpoi = d['K']?d['K']:""
def b_subpoi = d['AB']?d['AB']:""
if(a_subpoi == b_subpoi){
subpoiCount.addAndGet(1)
sameCount++
}else{
list << "subpoi"
}
def a_houseno = d['L']?d['L'].replace("号楼","").replace("幢楼","").replace("栋号","").replace("幢","").replace("栋","").replace("座","").replace("号","").replace("楼","").replace("第","").replace("楼",""):""
def b_houseno = d['AD']?d['AD'].replace("号楼","").replace("幢楼","").replace("栋号","").replace("幢","").replace("栋","").replace("座","").replace("号","").replace("楼","").replace("第","").replace("楼",""):""
if(a_houseno == b_houseno){
housenoCount.addAndGet(1)
sameCount++
}else{
list << "houseno"
}
def a_cellno = d['M']?chNumberConver(d['M'].replace("单元","")):""
def b_cellno = d['AE']?chNumberConver(d['AE'].replace("单元","")):""
if(a_cellno == b_cellno){
cellnoCount.addAndGet(1)
sameCount++
}else{
list << "cellno"
}
def a_floorno = d['N']?d['N'].replace("层楼","").replace("层","").replace("楼",""):""
def b_floorno = d['AF']?d['AF'].replace("层楼","").replace("层","").replace("楼",""):""
if(a_floorno == b_floorno){
floornoCount.addAndGet(1)
sameCount++
}else{
list << "floorno"
}
def a_roomno = d['O']?d['O'].replace("室号","").replace("号门","").replace("室","").replace("房","").replace("户","").replace("号","").replace("门",""):""
def b_roomno = d['AG']?d['AG'].replace("室号","").replace("号门","").replace("室","").replace("房","").replace("户","").replace("号","").replace("门",""):""
if(a_roomno == b_roomno){
roomnoCount.addAndGet(1)
sameCount++
}else{
list << "roomno"
}
def a_dev = d['Q']?d['Q']:''
def b_dev = d['AH']?d['AH']:''
if(a_dev == b_dev){
devCount.addAndGet(1)
sameCount++
}else{
list << "dev"
}
d['AI'] = sameCount
d['AJ'] = list.join(",")
},0)
def totalCount = total.get()
println totalCount
def prov_pre = DataHandleUtils.getDoubleRound(2, provCount.get()*100.0/totalCount)
def city_pre = DataHandleUtils.getDoubleRound(2, cityCount.get()*100.0/totalCount)
def dist_pre = DataHandleUtils.getDoubleRound(2, distCount.get()*100.0/totalCount)
def town_pre = DataHandleUtils.getDoubleRound(2, townCount.get()*100.0/totalCount)
def community_pre = DataHandleUtils.getDoubleRound(2, communityCount.get()*100.0/totalCount)
def road_pre = DataHandleUtils.getDoubleRound(2, roadCount.get()*100.0/totalCount)
def roadno_pre = DataHandleUtils.getDoubleRound(2, roadnoConnt.get()*100.0/totalCount)
def poi_pre = DataHandleUtils.getDoubleRound(2, poiCount.get()*100.0/totalCount)
def subpoi_pre = DataHandleUtils.getDoubleRound(2, subpoiCount.get()*100.0/totalCount)
def houseno_pre = DataHandleUtils.getDoubleRound(2, housenoCount.get()*100.0/totalCount)
def cellno_pre = DataHandleUtils.getDoubleRound(2, cellnoCount.get()*100.0/totalCount)
def floor_pre = DataHandleUtils.getDoubleRound(2, floornoCount.get()*100.0/totalCount)
def roomno_pre = DataHandleUtils.getDoubleRound(2, roomnoCount.get()*100.0/totalCount)
def datas = []
datas << ["总量", "prov","city","dist","town","community","road","roadno","poi","subpoi","houseno","cellno","floor","roomno"]
datas << [totalCount,prov_pre,city_pre,dist_pre,town_pre,community_pre,road_pre,roadno_pre,poi_pre,subpoi_pre,houseno_pre,cellno_pre,floor_pre,roomno_pre]
ExcelUtils.write("d://数据//地址标准化//测试结果1.xlsx", datas, "数据")
}
def chNumberConver(def s){
if(s && s.size()==1){
return s.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
}
return s;
}
}
@@ -15,8 +15,8 @@ class CrawlDistrictIdSelect {
static main(args) {
def task = new CrawlDistrictIdSelect()
// task.run()
task.synch()
task.run()
// task.synch()
}
def synch(){
@@ -38,25 +38,34 @@ class CrawlDistrictIdSelect {
def run(){
def datas = findCrawlData()
def list = []
datas.each {k,v->
if(v){
for(def d : v){
d.c_date = new Date()
d.m_date = new Date()
districtseedsColl.save(d)
list << d
}
}
if(list.size()==10000){
districtseedsColl.insert(list)
list = []
}
}
if(list.size()>0){
districtseedsColl.insert(list)
}
}
def findCrawlData(){
def q = new BasicDBObject()
// q.put("city","福州")
q.put("source", "诸葛找房")
q.put("s_date",202010)
q.put("source", "安居客")
q.put("s_date",202012)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = "xiaoqu.zhuge.com/"
def prefix = "anjuke.com/"
def keyList =new HashSet()
for(def db : dbcursor){
def url = null
@@ -68,11 +77,11 @@ class CrawlDistrictIdSelect {
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com"))
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com"))
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
@@ -83,7 +92,7 @@ class CrawlDistrictIdSelect {
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "zhuge.com"
map.root = "anjuke.com"
map.city_short= shortCity
map.city = db.city
map.region = db.region
@@ -4,14 +4,17 @@ import com.fangjia.script.util.CommonUtils
import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash
/**
* 福州农商行
* 将抓取的小区均价同步到第三方均价表中
*/
class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋"]
//"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋"
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -19,13 +22,14 @@ class SynchDistrictPrice {
def run(){
def citys = ["福州"]//CommonUtils.findAllCity()
def citys = CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
def rets = findCrawlDistrictDetails(city, k, v)
synchData(city, rets, v)
println city +" "+rets.size()
synchData(city, rets, v)
}
}
@@ -76,7 +80,7 @@ class SynchDistrictPrice {
}catch(Exception e){
println e.getMessage()
}
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
}else{
for(def r : rets){
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
@@ -85,9 +89,10 @@ class SynchDistrictPrice {
thirdPartyColl.save(r)
}
}
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
}
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
}
def findIsExists(def city, def region, def name, def sDate, def source){
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202102);
def count2 = findTotalCount(source, 202103);
def count1 = findTotalCount(source, 202105);
def count2 = findTotalCount(source, 202106);
println source +" "+count1 +" "+count2
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.util
import com.fangjia.script.lixue.util.DateFormat
import org.apache.commons.lang.StringUtils
import org.apache.poi.hssf.record.BoolErrRecord
@@ -389,8 +390,8 @@ public class ExcelUtils{
try{
closure.call(row)
int count = counter.incrementAndGet()
if(count % 100 == 0){
println("执行进度:${com.fangjia.core.util.DataUtil.roundDouble((count * 100)/total, 2)}%")
if(count % 1000 == 0){
println(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+"执行进度:${com.fangjia.core.util.DataUtil.roundDouble((count * 100)/total, 2)}%")
}
}catch (Exception e){
println(row)
@@ -121,9 +121,9 @@ public class OpenApiClient {
params.put("suggestName", suggestName)
}
String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/address/matchV2", params, "utf-8")
while (content == null) {
content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8")
content = new HttpRequest().get("http:///open.fangjia.com/address/matchV2", params, "utf-8")
println("${city} ${input} ${cat}程序调用异常")
Thread.sleep(6000)
}
@@ -173,6 +173,41 @@ class GaoDeApiUtils {
}
def static around2(String location,String radius, String keywords = null, types = null) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
if(keywords){
params.put("keywords", keywords)
}
if(types){
params.put("types", types)
}
if(location){
params.put("location", location)
}
if(radius){
params.put("radius", radius)
}
params.put("sortrule", "distance")
String content = null;
try {
content = new HttpRequest().newGet("https://restapi.amap.com/v3/place/around", params, "utf-8")
} catch (Exception e) {
return null;
}
JSONObject jsonObj = JSONObject.fromObject(content)
if(jsonObj && jsonObj.get("infocode").equals("10000")){
return jsonObj
}else if(jsonObj && jsonObj.get("infocode").equals("20003")){
return null;
} else {
return around(location, radius, keywords, types)
}
return null;
}
def static searchKey(def keyword, def cityCode){
def url = "https://www.amap.com/service/poiInfo?query_type=TQUERY&pagesize=20&pagenum=1&qii=true&cluster_state=5&need_utd=true&utd_sceneid=1000&div=PC1000&addr_poi_merge=true&is_classify=true&zoom=10&city="+cityCode+"&geoobj=113.082026%7C22.516777%7C115.059608%7C22.922141&keywords="+URLEncoder.encode(keyword,"UTF-8")
String content