diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c2f8fe431..6cd17549e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -27,7 +27,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{ }else{ def jsonObj = null if(dist.address){ - jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) +// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) + jsonObj = OpenApiClient.standard1Action(dist.city, dist.address) } - if(jsonObj && jsonObj.searchAddress){ - ret << jsonObj.searchAddress.street - ret << jsonObj.searchAddress.streetNumber + if(jsonObj && jsonObj.street){ + if(jsonObj.street){ + ret << jsonObj.street + }else{ + ret << "" + } + + if(jsonObj.streetNumber){ + ret << jsonObj.streetNumber + }else{ + ret << "" + } + }else{ ret << "" ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index d554c0376..7e5074c7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -137,7 +137,7 @@ ExcelUtils.write("d:\\ def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" println s def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" -//DingDingMessageUtil.sendTextMessage(access_token, s) +DingDingMessageUtil.sendTextMessage(access_token, s) ////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 77239e3ea..e4b621c99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -164,6 +164,7 @@ class OpenApiUtils { if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ return null; } + address = address.trim().replace(" ","") String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" List poiList = null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy index c60a811b5..19709dd22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy @@ -18,44 +18,62 @@ class CcbCityPriceStat { task.run() } def run(){ - def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] def rets = [] + def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd") + for(def city : citys){ - def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) - def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30")) - def cityRets = [] - for(def dist : districts){ - def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01") - def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) - if(tradeAvgPrice && weekendPrice){ - def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] - cityRets << row - println row.toString() + def matchTotalCount = findMatchTradeCount(city, startDate, endDate) + println city +" "+matchTotalCount + if(matchTotalCount){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29")) + def cityRets = [] + for(def dist : districts){ + def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01") + def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) + if(tradeAvgPrice && weekendPrice){ + def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] + cityRets << row + println row.toString() + } } + print city +" "+ cityRets.size() + rets.addAll(cityRets) } - print city +" "+ cityRets.size() - rets.addAll(cityRets) + } println "总量:"+rets.size() - ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有") + ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有") + } + + + def findMatchTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def count = DEAL_PRICE_COLL.count(q) + return count } def findTrades(dist, startDate, endDate){ def q = new BasicDBObject() - q.put("city", dist.city) - q.put("region", dist.region) - - if(dist.alias){ - if(!dist.alias.contains(dist.name)){ - dist.alias << dist.name - } - q.put("district_name", new BasicDBObject('$in', dist.alias)) - }else{ - q.put("district_name", dist.name) - } - +// q.put("city", dist.city) +// q.put("region", dist.region) +// +// if(dist.alias){ +// if(!dist.alias.contains(dist.name)){ +// dist.alias << dist.name +// } +// q.put("district_name", new BasicDBObject('$in', dist.alias)) +// }else{ +// q.put("district_name", dist.name) +// } + q.put("match_id", dist._id.toString()) q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) def dbcorsor = DEAL_PRICE_COLL.find(q) def avg_price = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy new file mode 100644 index 000000000..02fc9458d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy @@ -0,0 +1,347 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.DateUtils +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * 鎴愪氦鏁版嵁瑙f瀽宸ュ叿绫� + */ +class DealPriceUtils { + + static final Map DEAL_DATE_RESOLVERS = [:] + static final Map ROOM_RESOLVERS = [:] + static final Map HALL_RESOLVERS = [:] + static final Map TOILET_RESOLVERS = [:] + static final Map TOTAL_PRICE_RESOLVERS = [:] + static final Map UNIT_PRICE_RESOLVERS = [:] + static final Map AREA_RESOLVERS = [:] + static final Map HEIGHT_RESOLVERS = [:] + static final Map FLOOR_RESOLVERS = [:] + static final List DIRECTIONS = ["涓滃寳", "瑗垮寳", "涓滃崡", "瑗垮崡", "鍗楀寳", "涓滆タ", "涓�", "鍗�", "瑗�", "鍖�"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + + static { + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd") + }) + + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM") + }) + + ROOM_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓(瀹鎴�)"), { String str -> + str = str.replaceAll("瀹鎴�", '').trim() + return str.toInteger() + }) + + HALL_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍘�"), { String str -> + str = str.replace("鍘�", '').trim() + return str.toInteger() + }) + TOILET_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍗�"), { String str -> + str = str.replace("鍗�", '').trim() + return str.toInteger() + }) + TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?涓�"), { String str -> + str = str.replace("涓�", '').trim() + return (str.toDouble() * 10000).toInteger() + }) + + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/銕�"), { String str -> + str = str.replace("鍏�/銕�", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/m虏"), { String str -> + str = str.replace("鍏�/m虏", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/骞�"), { String str -> + str = str.replace("鍏�/骞�", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�"), { String str -> + str = str.replace("鍏�", '').trim() + return str.toDouble().toInteger() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞崇背"), { String str -> + str = str.replace("骞崇背", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞虫柟绫�"), { String str -> + str = str.replace("骞虫柟绫�", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞�"), { String str -> + str = str.replace("骞�", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?銕�"), { String str -> + str = str.replace("銕�", '').trim() + return str.toDouble() + }) + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m虏"), { String str -> + str = str.replace("m虏", '').trim() + return str.toDouble() + }) + HEIGHT_RESOLVERS.put(Pattern.compile("鍏盶\d+灞�"), { String str -> + str = str.replace("鍏�", '').replace('灞�', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("鎬籠\d+灞�"), { String str -> + str = str.replace("鎬�", '').replace('灞�', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n2 + } + return null + }) + + + FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n1 + } + return n1 + }) + + FLOOR_RESOLVERS.put(Pattern.compile("绗琝\d+(灞倈妤�)"), { String str -> + str = str.replace("绗�", "").replaceAll("灞倈妤�", "") + return str.toInteger() + }) + + FITMENTS_TYPES_MAPS.put("姣涘澂", ["姣涘澂", "娓呮按", "鏃犺淇�", "鐙爧姣涘澂", "姣涜儦", "鐧藉澂", "鏃ц", "鍧�", "鍘熸埧", "姣涘潖"]) + FITMENTS_TYPES_MAPS.put("绠�瑁�", ["绠�瑁�", "鏅�氳淇�", "鏅��", "绠�鍗曡淇�", "绠�鏄撹淇�", "鏅", "涓�鑸淇�", "涓�鑸�", "绠�", "绮楄淇�"]) + FITMENTS_TYPES_MAPS.put("绮捐", ["绮捐", "鍏ㄨ淇�", "鍏ㄨ", "涓瓑瑁呬慨", "涓", "涓。瑁呬慨", "绮捐嚧瑁呬慨", "涓。", "涓瓑", "绮�", "涓慨", "浼樿川瑁呬慨", "鏃跺皻瑁呬慨", "鑹", "涓珮鎸�"]) + FITMENTS_TYPES_MAPS.put("璞", ["璞崕瑁呬慨", "楂樻。", "璞崕", "璞", "楂樼骇瑁呬慨", "楂樿", "濂緢"]) + + } + /** + * 瑙f瀽鎴愪氦鏃ユ湡 + * @param content + * @return + */ + static Date parseDealDate(String content) { + return parse(content, DEAL_DATE_RESOLVERS) + } + + /** + * 瑙f瀽瀹ゆ暟 + * @param content + * @return + */ + static Integer parseRoom(String content) { + return parse(content, ROOM_RESOLVERS) + } + /** + * 瑙f瀽鍘呮暟 + * @param content + * @return + */ + static Integer parseHall(String content) { + return parse(content, HALL_RESOLVERS) + } + /** + * 瑙f瀽鍘呮暟 + * @param content + * @return + */ + static Integer parseToilet(String content) { + return parse(content, TOILET_RESOLVERS) + } + + /** + * 瑙f瀽鎬讳环 + * @param content + * @return + */ + static Integer parseTotalPrice(String content) { + return parse(content, TOTAL_PRICE_RESOLVERS) + } + /** + * 瑙f瀽鍗曚环 + * @param content + * @return + */ + static Integer parseUnitPrice(String content) { + return parse(content, UNIT_PRICE_RESOLVERS) + } + /** + * 瑙f瀽闈㈢Н + * @param content + * @return + */ + static Double parseArea(String content) { + return parse(content, AREA_RESOLVERS) + } + /** + * 瑙f瀽妤煎眰 + * @param content + * @return + */ + static Integer parseFloor(String content) { + return parse(content, FLOOR_RESOLVERS) + } + /** + * 瑙f瀽妤奸珮 + * @param content + * @return + */ + static Integer parseHeight(String content) { + return parse(content, HEIGHT_RESOLVERS) + } + + /** + * 鏍煎紡鍖栧皬鍖哄悕 + * @param name + * @return + */ + static String formatName(String name) { + name = name.replace("-鍟嗕綇", "").replace("-鍒", "") + //姹囩瀹跺洯骞冲畨閲岋紙杩侊級2015-2017 + name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "") + //鍥借锤澶╂偊涓�鏈�2017 + name = name.replaceAll("\\d\\d\\d\\d\$", "") + if (name ==~ /.+(\(|锛�)[\d涓�浜屼笁鍥涗簲鍏竷鍏節鍗佷笢鍗楄タ鍖梶\w]+(鏈焲鍖�)(\)|锛�)$/) { + name = name.replaceAll("\\(|锛坾\\)|锛�", '') + } else if (name ==~ /.+(\(|锛�)鍒(\)|锛�)$/) { + name = name.replaceAll("\\(|锛坾\\)|锛�", '') + } else if (name ==~ /.+(\(|锛�).+(\)|锛�)$/) { + name = name.replaceAll("(\\(|锛�).+(\\)|锛�)", '') + } else if (name.contains("路") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) { + println("---------------------------->" + name) + name = name.replaceAll("路|-", '') + println(name) + } + if (name ==~ /.{2,}璺�.{3,}$/ && !(name ==~ /.{2,}璺�.+鍙�/) && !(name ==~ /.{2,}璺�.+寮�/) && !(name ==~ /.*灏忓尯$/)) { + name = name.replaceAll("^.{2,}璺�", "") + } + return name + } + + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("涓�","1").replace("浜�","2").replace("涓�","3").replace("鍥�","4").replace("浜�","5").replace("涓�","2") + return resolver.getValue()(match) + } + } + return null + } + + /** + * 鍖归厤绗﹀悎鏉′欢鐨勫瓧绗︿覆 + * + * @param str + * @param pattern + * @return + */ + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("闈炴瘺鍧�")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|锛坾銆妡\"|鈥渱銆恷銆�).+?(\\)|銆媩锛墊\"|鈥潀銆憒銆�)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + + static String parsePrice(String priceStr) { + def pattern =Pattern.compile("\\d+(\\.\\d+)?涓�") + String match = match(priceStr, pattern) + } + + static void main(String[] args) { + println parsePrice("椤剁娆e洯瑗胯嫅2瀹�1鍘�1鍗�492.3涓�") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy new file mode 100644 index 000000000..845ab47d0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy @@ -0,0 +1,167 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.thirdPriceCrawl.Utils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class ZhugeFangDistCrawl { + + public static final SHORT_DATE = Utils.getLastMonth(); + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new ZhugeFangDistCrawl(); + task.run(); + } + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/" + WebDriver driver = new ChromeDriver() + driver.manage().deleteAllCookies() + getRegions(driver, url) + } + + def getRegions(driver, url) { + for(def i=1;i<=11;i++){ + def newUrl = url+"page${i}/" + println newUrl + def doc = getDocument(newUrl, driver) + def total = parseList(doc) + +// def next = JQueryUtils.select("a:contains(下一页)", doc) +// if(next){ +// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() +// } + + } + + } + + def parseList(doc){ + def city= "福州" + def els = JQueryUtils.select("#listTableBox li", doc) + if (els) { + for(def e : els){ + def name = e.select(".house-name").text() + def str = e.select(".house-adr").text() + def address = null + if(str){ + if(str.contains("]")){ + address = str.substring(str.indexOf("]")+1).trim() + }else{ + address = str + } + } + def regionStr = null + if(str && str.contains("]") && str.contains("[")){ + regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]")) + } + + def region = null + def block = null + if(regionStr){ + def arrs = regionStr.split("-") + if(arrs && arrs.size()>0){ + if(arrs.size()==2){ + block = arrs[1].trim() + } + if(arrs.size()>=1){ + region = arrs[0].trim() + } + + } + } + def links = e.select(".house-name a") + def href = null + if(links){ + href = links[0].attr("href") + } + def priceStr = e.select(".price-number").text() + def price = null + if(priceStr && !priceStr.contains("暂无")){ + if(priceStr.contains("元")){ + priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim() + } + if(priceStr && priceStr.isNumber()){ + price = priceStr.toInteger() + } + } + def obj = findByParams(city, region, name) + if(obj){ + if(price){ + DBObject udata2 = new BasicDBObject() + udata2.put("avg_price", price) + udata2.put("m_date", new Date()) + println "[upeate] ${region} "+name + thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2)); + } + }else{ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + udata.put("block", block) + udata.put("url", href) + udata.put("address", address) + udata.put("name", name) + udata.put("avg_price", price) + udata.put("s_date", SHORT_DATE) + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + udata.put("source", "诸葛找房") + thirdPartyColl.save(udata) + println "[save] ${region} "+name+" "+priceStr + } + } + } + return els.size() + } + + + def findByParams(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", "诸葛找房") + q.put("s_date", SHORT_DATE) + return thirdPartyColl.findOne(q); + } + + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(10000) + def els = JQueryUtils.select("#listTableBox li", doc) + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy new file mode 100644 index 000000000..47c9fe748 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy @@ -0,0 +1,123 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils + +class DistrictMapMatch { + + static main(args) { + def task = new DistrictMapMatch() + task.mapMatch() + } + + + def mapMatch(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d-> + if(!d['H']){ + index++ + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// if (!map || !map.lat) { +// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); +// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// } + if(map){ + d['F'] = map.id + if(map.adname){ + d['G'] = map.adname + }else if(map.region){ + d['G'] = map.region + } + d['H'] = map.name + d['I'] = map.address + if(map.type){ + d['J'] = map.type + }else if(map.estateType){ + d['J'] = map.estateType + } + d['K'] = map.lng + d['L'] = map.lat + d['M'] = map.score + d['N'] = map.source + } + } + },0) + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 95bfde522..fe522570a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -945,20 +945,20 @@ class CZDataBatch extends BaseBatch{ if(price){ return price.avg_price; }else{ - //样例估价 - def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false); - try{ - JSONObject jsonObject = JSONObject.fromObject(s); - def map = DataHandleUtils.JsonToMap(jsonObject); - if(map.code==200){ - if(map.result){ - return map.result.avgPrice; - } - - } - }catch(Exception e){ - - } +// //样例估价 +// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false); +// try{ +// JSONObject jsonObject = JSONObject.fromObject(s); +// def map = DataHandleUtils.JsonToMap(jsonObject); +// if(map.code==200){ +// if(map.result){ +// return map.result.avgPrice; +// } +// +// } +// }catch(Exception e){ +// +// } //分期找总期 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 011f434e3..9a9fd90ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -30,7 +30,7 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger import java.util.regex.Matcher -import java.util.regex.Patter +import java.util.regex.Pattern /** * 第三方数据标准化处理 @@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{ def filterSpacialChar(def s){ def pattern = ~/[()!;\·]\[【】()]/ def matcher = pattern.matcher(s) + def count = matcher.getCount() if(count && count>0){ return matcher.replaceAll('_') diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index fe433f087..21887e3db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -169,7 +169,7 @@ public class CommonUtils { static void sendMessage(String queueName, byte[] bytes){ ConnectionFactory factory = new ConnectionFactory(); - factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f"); + factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f"); Connection conn = factory.newConnection(); Channel channel = conn.createChannel(); channel.queueDeclare(queueName, true, false, false, null); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index f0186187b..3d4e855f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -348,6 +348,34 @@ public class OpenApiClient { } + public static JSONObject standard1Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standardize(String city, String address) { try { Map params = new HashMap();