update
This commit is contained in:
+16
-5
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
@@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}else{
|
||||
def jsonObj = null
|
||||
if(dist.address){
|
||||
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
|
||||
}
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
if(jsonObj && jsonObj.street){
|
||||
if(jsonObj.street){
|
||||
ret << jsonObj.street
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(jsonObj.streetNumber){
|
||||
ret << jsonObj.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
|
||||
@@ -137,7 +137,7 @@ ExcelUtils.write("d:\\
|
||||
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
|
||||
println s
|
||||
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
//DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
|
||||
|
||||
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
|
||||
|
||||
@@ -164,6 +164,7 @@ class OpenApiUtils {
|
||||
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
|
||||
return null;
|
||||
}
|
||||
address = address.trim().replace(" ","")
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
|
||||
|
||||
List<com.alibaba.fastjson.JSONObject> poiList = null;
|
||||
|
||||
+44
-26
@@ -18,44 +18,62 @@ class CcbCityPriceStat {
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
def rets = []
|
||||
def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")
|
||||
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
|
||||
def cityRets = []
|
||||
for(def dist : districts){
|
||||
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
|
||||
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
|
||||
if(tradeAvgPrice && weekendPrice){
|
||||
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
|
||||
cityRets << row
|
||||
println row.toString()
|
||||
def matchTotalCount = findMatchTradeCount(city, startDate, endDate)
|
||||
println city +" "+matchTotalCount
|
||||
if(matchTotalCount){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29"))
|
||||
def cityRets = []
|
||||
for(def dist : districts){
|
||||
def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01")
|
||||
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
|
||||
if(tradeAvgPrice && weekendPrice){
|
||||
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
|
||||
cityRets << row
|
||||
println row.toString()
|
||||
}
|
||||
}
|
||||
print city +" "+ cityRets.size()
|
||||
rets.addAll(cityRets)
|
||||
}
|
||||
print city +" "+ cityRets.size()
|
||||
rets.addAll(cityRets)
|
||||
|
||||
}
|
||||
|
||||
println "总量:"+rets.size()
|
||||
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
|
||||
ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有")
|
||||
}
|
||||
|
||||
|
||||
def findMatchTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', true))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
// q.put("city", dist.city)
|
||||
// q.put("region", dist.region)
|
||||
//
|
||||
// if(dist.alias){
|
||||
// if(!dist.alias.contains(dist.name)){
|
||||
// dist.alias << dist.name
|
||||
// }
|
||||
// q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
// }else{
|
||||
// q.put("district_name", dist.name)
|
||||
// }
|
||||
q.put("match_id", dist._id.toString())
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 成交数据解析工具类
|
||||
*/
|
||||
class DealPriceUtils {
|
||||
|
||||
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
|
||||
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
|
||||
static {
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
|
||||
})
|
||||
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
|
||||
})
|
||||
|
||||
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
|
||||
str = str.replaceAll("室|房", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
|
||||
str = str.replace("厅", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
|
||||
str = str.replace("卫", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
|
||||
str = str.replace("万", '').trim()
|
||||
return (str.toDouble() * 10000).toInteger()
|
||||
})
|
||||
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
|
||||
str = str.replace("元/㎡", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m²"), { String str ->
|
||||
str = str.replace("元/m²", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
|
||||
str = str.replace("元/平", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
|
||||
str = str.replace("元", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
|
||||
str = str.replace("平米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
|
||||
str = str.replace("平方米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
|
||||
str = str.replace("平", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
|
||||
str = str.replace("㎡", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m²"), { String str ->
|
||||
str = str.replace("m²", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
|
||||
str = str.replace("共", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
|
||||
str = str.replace("总", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n2
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n1
|
||||
}
|
||||
return n1
|
||||
})
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
|
||||
str = str.replace("第", "").replaceAll("层|楼", "")
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
|
||||
}
|
||||
/**
|
||||
* 解析成交日期
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Date parseDealDate(String content) {
|
||||
return parse(content, DEAL_DATE_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析室数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseRoom(String content) {
|
||||
return parse(content, ROOM_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHall(String content) {
|
||||
return parse(content, HALL_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseToilet(String content) {
|
||||
return parse(content, TOILET_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析总价
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseTotalPrice(String content) {
|
||||
return parse(content, TOTAL_PRICE_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析单价
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseUnitPrice(String content) {
|
||||
return parse(content, UNIT_PRICE_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析面积
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Double parseArea(String content) {
|
||||
return parse(content, AREA_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析楼层
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseFloor(String content) {
|
||||
return parse(content, FLOOR_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析楼高
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHeight(String content) {
|
||||
return parse(content, HEIGHT_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化小区名
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
static String formatName(String name) {
|
||||
name = name.replace("-商住", "").replace("-别墅", "")
|
||||
//汇福家园平安里(迁)2015-2017
|
||||
name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "")
|
||||
//国贸天悦一期2017
|
||||
name = name.replaceAll("\\d\\d\\d\\d\$", "")
|
||||
if (name ==~ /.+(\(|()[\d一二三四五六七八九十东南西北|\w]+(期|区)(\)|))$/) {
|
||||
name = name.replaceAll("\\(|(|\\)|)", '')
|
||||
} else if (name ==~ /.+(\(|()别墅(\)|))$/) {
|
||||
name = name.replaceAll("\\(|(|\\)|)", '')
|
||||
} else if (name ==~ /.+(\(|().+(\)|))$/) {
|
||||
name = name.replaceAll("(\\(|().+(\\)|))", '')
|
||||
} else if (name.contains("·") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) {
|
||||
println("---------------------------->" + name)
|
||||
name = name.replaceAll("·|-", '')
|
||||
println(name)
|
||||
}
|
||||
if (name ==~ /.{2,}路.{3,}$/ && !(name ==~ /.{2,}路.+号/) && !(name ==~ /.{2,}路.+弄/) && !(name ==~ /.*小区$/)) {
|
||||
name = name.replaceAll("^.{2,}路", "")
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
|
||||
static Object parse(String content, Map<Pattern, Closure> resolvers) {
|
||||
if (content != null) {
|
||||
content = content.replaceAll("\\s", "")
|
||||
}
|
||||
for (def resolver : resolvers.entrySet()) {
|
||||
String match = match(content, resolver.getKey())
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
|
||||
return resolver.getValue()(match)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配符合条件的字符串
|
||||
*
|
||||
* @param str
|
||||
* @param pattern
|
||||
* @return
|
||||
*/
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
static String parsePrice(String priceStr) {
|
||||
def pattern =Pattern.compile("\\d+(\\.\\d+)?万")
|
||||
String match = match(priceStr, pattern)
|
||||
}
|
||||
|
||||
static void main(String[] args) {
|
||||
println parsePrice("顶秀欣园西苑2室1厅1卫492.3万")
|
||||
}
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.Utils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class ZhugeFangDistCrawl {
|
||||
|
||||
public static final SHORT_DATE = Utils.getLastMonth();
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
def task = new ZhugeFangDistCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.manage().deleteAllCookies()
|
||||
getRegions(driver, url)
|
||||
}
|
||||
|
||||
def getRegions(driver, url) {
|
||||
for(def i=1;i<=11;i++){
|
||||
def newUrl = url+"page${i}/"
|
||||
println newUrl
|
||||
def doc = getDocument(newUrl, driver)
|
||||
def total = parseList(doc)
|
||||
|
||||
// def next = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
// if(next){
|
||||
// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def parseList(doc){
|
||||
def city= "福州"
|
||||
def els = JQueryUtils.select("#listTableBox li", doc)
|
||||
if (els) {
|
||||
for(def e : els){
|
||||
def name = e.select(".house-name").text()
|
||||
def str = e.select(".house-adr").text()
|
||||
def address = null
|
||||
if(str){
|
||||
if(str.contains("]")){
|
||||
address = str.substring(str.indexOf("]")+1).trim()
|
||||
}else{
|
||||
address = str
|
||||
}
|
||||
}
|
||||
def regionStr = null
|
||||
if(str && str.contains("]") && str.contains("[")){
|
||||
regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]"))
|
||||
}
|
||||
|
||||
def region = null
|
||||
def block = null
|
||||
if(regionStr){
|
||||
def arrs = regionStr.split("-")
|
||||
if(arrs && arrs.size()>0){
|
||||
if(arrs.size()==2){
|
||||
block = arrs[1].trim()
|
||||
}
|
||||
if(arrs.size()>=1){
|
||||
region = arrs[0].trim()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def links = e.select(".house-name a")
|
||||
def href = null
|
||||
if(links){
|
||||
href = links[0].attr("href")
|
||||
}
|
||||
def priceStr = e.select(".price-number").text()
|
||||
def price = null
|
||||
if(priceStr && !priceStr.contains("暂无")){
|
||||
if(priceStr.contains("元")){
|
||||
priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim()
|
||||
}
|
||||
if(priceStr && priceStr.isNumber()){
|
||||
price = priceStr.toInteger()
|
||||
}
|
||||
}
|
||||
def obj = findByParams(city, region, name)
|
||||
if(obj){
|
||||
if(price){
|
||||
DBObject udata2 = new BasicDBObject()
|
||||
udata2.put("avg_price", price)
|
||||
udata2.put("m_date", new Date())
|
||||
println "[upeate] ${region} "+name
|
||||
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
|
||||
}
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", city)
|
||||
udata.put("region", region)
|
||||
udata.put("block", block)
|
||||
udata.put("url", href)
|
||||
udata.put("address", address)
|
||||
udata.put("name", name)
|
||||
udata.put("avg_price", price)
|
||||
udata.put("s_date", SHORT_DATE)
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("source", "诸葛找房")
|
||||
thirdPartyColl.save(udata)
|
||||
println "[save] ${region} "+name+" "+priceStr
|
||||
}
|
||||
}
|
||||
}
|
||||
return els.size()
|
||||
}
|
||||
|
||||
|
||||
def findByParams(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("source", "诸葛找房")
|
||||
q.put("s_date", SHORT_DATE)
|
||||
return thirdPartyColl.findOne(q);
|
||||
}
|
||||
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
Thread.sleep(10000)
|
||||
def els = JQueryUtils.select("#listTableBox li", doc)
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class DistrictMapMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictMapMatch()
|
||||
task.mapMatch()
|
||||
}
|
||||
|
||||
|
||||
def mapMatch(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d->
|
||||
if(!d['H']){
|
||||
index++
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// if (!map || !map.lat) {
|
||||
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// }
|
||||
if(map){
|
||||
d['F'] = map.id
|
||||
if(map.adname){
|
||||
d['G'] = map.adname
|
||||
}else if(map.region){
|
||||
d['G'] = map.region
|
||||
}
|
||||
d['H'] = map.name
|
||||
d['I'] = map.address
|
||||
if(map.type){
|
||||
d['J'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['J'] = map.estateType
|
||||
}
|
||||
d['K'] = map.lng
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.score
|
||||
d['N'] = map.source
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
}
|
||||
+14
-14
@@ -945,20 +945,20 @@ class CZDataBatch extends BaseBatch{
|
||||
if(price){
|
||||
return price.avg_price;
|
||||
}else{
|
||||
//样例估价
|
||||
def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
|
||||
try{
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
def map = DataHandleUtils.JsonToMap(jsonObject);
|
||||
if(map.code==200){
|
||||
if(map.result){
|
||||
return map.result.avgPrice;
|
||||
}
|
||||
|
||||
}
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
// //样例估价
|
||||
// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
|
||||
// try{
|
||||
// JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
// def map = DataHandleUtils.JsonToMap(jsonObject);
|
||||
// if(map.code==200){
|
||||
// if(map.result){
|
||||
// return map.result.avgPrice;
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }catch(Exception e){
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
//分期找总期
|
||||
|
||||
+2
-1
@@ -30,7 +30,7 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Patter
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 第三方数据标准化处理
|
||||
@@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def filterSpacialChar(def s){
|
||||
def pattern = ~/[()!;\·]\[【】()]/
|
||||
def matcher = pattern.matcher(s)
|
||||
|
||||
def count = matcher.getCount()
|
||||
if(count && count>0){
|
||||
return matcher.replaceAll('_')
|
||||
|
||||
@@ -169,7 +169,7 @@ public class CommonUtils {
|
||||
|
||||
static void sendMessage(String queueName, byte[] bytes){
|
||||
ConnectionFactory factory = new ConnectionFactory();
|
||||
factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f");
|
||||
factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f");
|
||||
Connection conn = factory.newConnection();
|
||||
Channel channel = conn.createChannel();
|
||||
channel.queueDeclare(queueName, true, false, false, null);
|
||||
|
||||
@@ -348,6 +348,34 @@ public class OpenApiClient {
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject standard1Action(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("token", TOKEN);
|
||||
params.put("__refresh_cache","1")
|
||||
|
||||
String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8")
|
||||
println("${city} ${address} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if (result.getInt("code") == 200) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${city} ${address}")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${address} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject standardize(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
|
||||
Reference in New Issue
Block a user