Merge branch 'script_gj' into 'master'
Script gj See merge request !320
This commit is contained in:
+16
-5
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
@@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}else{
|
||||
def jsonObj = null
|
||||
if(dist.address){
|
||||
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
|
||||
}
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
if(jsonObj && jsonObj.street){
|
||||
if(jsonObj.street){
|
||||
ret << jsonObj.street
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(jsonObj.streetNumber){
|
||||
ret << jsonObj.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
class CcbDataStat {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDataStat()
|
||||
// task.run()
|
||||
task.tradeRun()
|
||||
}
|
||||
|
||||
def tradeRun(){
|
||||
def ccbCitys = CcbCommonUtils.citys;
|
||||
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d ->
|
||||
if(d['A']){
|
||||
if(ccbCitys.contains(d['A'])){
|
||||
d['D'] = findOracleTotalCount(d['A'])
|
||||
}else{
|
||||
def totalcount = findAllTradeCount(d['A'], startDate, endDate)
|
||||
def repeatCount = findRepeatCount(d['A'], startDate, endDate)
|
||||
def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate)
|
||||
def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate)
|
||||
d['B'] = totalcount
|
||||
d['C'] = repeatCount
|
||||
d['D'] = matchTotalCount
|
||||
d['E'] = matchRepeatTotalCount
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findRepeatCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def set = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
|
||||
if(!set.contains(key)){
|
||||
set << key
|
||||
}
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findMatchRepeatTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', true))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def set = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
|
||||
if(!set.contains(key)){
|
||||
set << key
|
||||
}
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
|
||||
def findMatchTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', true))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
public static def findAllTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
// q.put("match_id", new BasicDBObject('$exists', false))
|
||||
def total = DEAL_PRICE_COLL.count(q)
|
||||
return total
|
||||
}
|
||||
|
||||
def findOracleTotalCount(city){
|
||||
def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate<to_date('2022-01-01','yyyy-MM-dd')";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def run(){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
|
||||
if(d['A']) {
|
||||
def citycode = cityCodeDataMap.get(d['A'])
|
||||
def total = findTotalCount(citycode)
|
||||
def f_count = findDistFinishDate(citycode)
|
||||
def b_count = findBuildTypeCount(citycode)
|
||||
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
|
||||
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
|
||||
d['C'] = f_pre
|
||||
d['D'] = b_pre
|
||||
d['E'] = total
|
||||
println d['A'] +" "+ total +" "+f_count+" "+b_count
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findTotalCount(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findDistFinishDate(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findBuildTypeCount(citycode){
|
||||
def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
}
|
||||
@@ -137,7 +137,7 @@ ExcelUtils.write("d:\\
|
||||
def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )"
|
||||
println s
|
||||
def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60"
|
||||
//DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
|
||||
|
||||
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
|
||||
|
||||
@@ -164,6 +164,7 @@ class OpenApiUtils {
|
||||
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
|
||||
return null;
|
||||
}
|
||||
address = address.trim().replace(" ","")
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
|
||||
|
||||
List<com.alibaba.fastjson.JSONObject> poiList = null;
|
||||
|
||||
+44
-26
@@ -18,44 +18,62 @@ class CcbCityPriceStat {
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
def rets = []
|
||||
def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")
|
||||
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
|
||||
def cityRets = []
|
||||
for(def dist : districts){
|
||||
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
|
||||
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
|
||||
if(tradeAvgPrice && weekendPrice){
|
||||
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
|
||||
cityRets << row
|
||||
println row.toString()
|
||||
def matchTotalCount = findMatchTradeCount(city, startDate, endDate)
|
||||
println city +" "+matchTotalCount
|
||||
if(matchTotalCount){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29"))
|
||||
def cityRets = []
|
||||
for(def dist : districts){
|
||||
def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01")
|
||||
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
|
||||
if(tradeAvgPrice && weekendPrice){
|
||||
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
|
||||
cityRets << row
|
||||
println row.toString()
|
||||
}
|
||||
}
|
||||
print city +" "+ cityRets.size()
|
||||
rets.addAll(cityRets)
|
||||
}
|
||||
print city +" "+ cityRets.size()
|
||||
rets.addAll(cityRets)
|
||||
|
||||
}
|
||||
|
||||
println "总量:"+rets.size()
|
||||
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
|
||||
ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有")
|
||||
}
|
||||
|
||||
|
||||
def findMatchTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', true))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
// q.put("city", dist.city)
|
||||
// q.put("region", dist.region)
|
||||
//
|
||||
// if(dist.alias){
|
||||
// if(!dist.alias.contains(dist.name)){
|
||||
// dist.alias << dist.name
|
||||
// }
|
||||
// q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
// }else{
|
||||
// q.put("district_name", dist.name)
|
||||
// }
|
||||
q.put("match_id", dist._id.toString())
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class AnjukeHouseCrawl {
|
||||
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
|
||||
static main(args) {
|
||||
def task = new AnjukeHouseCrawl();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
for(def i=24;i<50;i++){
|
||||
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
|
||||
def doc = getDocument(url, driver)
|
||||
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
|
||||
println "页数:"+i+" 记录数:"+eles.size()
|
||||
if(eles){
|
||||
parser(eles, driver)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def parser(eles, driver){
|
||||
def index = 0
|
||||
for(def e : eles){
|
||||
index ++
|
||||
def href = e.attr("href")
|
||||
def doc = getDocument(href, driver)
|
||||
// parserHtml(doc)
|
||||
def newUrl = href.substring(0, href.indexOf("?"))
|
||||
sendMq(newUrl, doc)
|
||||
|
||||
println "[ "+index +"] "+ newUrl
|
||||
}
|
||||
}
|
||||
|
||||
def sendMq(href, doc){
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "sell_anjuke")
|
||||
json.put("url", href)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", doc.toString())
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
}
|
||||
|
||||
def parserHtml(doc) {
|
||||
String city = null
|
||||
def result = [:]
|
||||
result.siteCode = "安居客"
|
||||
if (doc) {
|
||||
result.title = JQueryUtils.selectText([".banner-title h1"], doc)
|
||||
def heads = JQueryUtils.select(".crumbs a", doc)
|
||||
|
||||
if (heads && heads.size() >= 5) {
|
||||
result.city = heads[0].text().replace("房产网", "")
|
||||
def orgRegion = heads[2].text().replace("二手房", "")
|
||||
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
|
||||
result.block = heads[3].text().replace("二手房", "")
|
||||
result.district_name = heads[4].text()
|
||||
} else if (heads && heads.size() >= 3) {
|
||||
result.city = heads[0].text().replace("房产网", "")
|
||||
result.district_name = heads[2].text()
|
||||
def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
|
||||
if (eles && eles.size() > 0) {
|
||||
def orgRegion = eles[0].text()
|
||||
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.district_name) {
|
||||
println " [未解析出(城市-区域-名称)信息]"
|
||||
}
|
||||
println result
|
||||
|
||||
|
||||
def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
|
||||
if (priceStr) {
|
||||
if (priceStr.isNumber()) {
|
||||
result.total_price = priceStr.toDouble() * 10000
|
||||
}
|
||||
}
|
||||
def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
|
||||
if (roomInfoStr) {
|
||||
Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
|
||||
Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
|
||||
Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
|
||||
|
||||
if (room) {
|
||||
result.room = room
|
||||
}
|
||||
if (hall) {
|
||||
result.hall = hall
|
||||
}
|
||||
if (toilet) {
|
||||
result.toilet = toilet
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
|
||||
if (areaStr) {
|
||||
result.area = areaStr.toDouble()
|
||||
}
|
||||
def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
|
||||
result.direction = DataCrawlUtils.parseDirection(directionStr)
|
||||
|
||||
def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
|
||||
if (buildDateStr && buildDateStr.contains("/")) {
|
||||
def arrs = buildDateStr.split("\\/")
|
||||
if (arrs && arrs.size() == 2) {
|
||||
result.build_date = arrs[0]
|
||||
result.estate_type = arrs[1]
|
||||
}
|
||||
}
|
||||
|
||||
result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
|
||||
if (result.floor_info && result.floor_info.contains("共")) {
|
||||
def height = result.floor_info
|
||||
if (result.floor_info.contains("(")) {
|
||||
height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
|
||||
}
|
||||
height = height.replace("共", "").replace("层", "")
|
||||
if (height) {
|
||||
result.height = height.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (result.area && result.total_price) {
|
||||
result.avg_price = (result.total_price / result.area).toInteger()
|
||||
}
|
||||
|
||||
def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
|
||||
result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
|
||||
|
||||
|
||||
def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
Thread.sleep(5000)
|
||||
def els = JQueryUtils.select(".list-main .list-left .list", doc)
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".props-main", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
}
|
||||
+250
@@ -0,0 +1,250 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataCrawlUtils {
|
||||
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
|
||||
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
|
||||
static {
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
|
||||
})
|
||||
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
|
||||
})
|
||||
|
||||
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
|
||||
str = str.replaceAll("室|房", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
|
||||
str = str.replace("厅", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
|
||||
str = str.replace("卫", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
|
||||
str = str.replace("万", '').trim()
|
||||
return (str.toDouble() * 10000).toInteger()
|
||||
})
|
||||
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
|
||||
str = str.replace("元/㎡", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
|
||||
str = str.replace("元/m?", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
|
||||
str = str.replace("元/平", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
|
||||
str = str.replace("元", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
|
||||
str = str.replace("平米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
|
||||
str = str.replace("平方米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
|
||||
str = str.replace("平", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
|
||||
str = str.replace("㎡", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
|
||||
str = str.replace("m?", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
|
||||
str = str.replace("共", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
|
||||
str = str.replace("总", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n2
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n1
|
||||
}
|
||||
return n1
|
||||
})
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
|
||||
str = str.replace("第", "").replaceAll("层|楼", "")
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析室数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseRoom(String content) {
|
||||
return parse(content, ROOM_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHall(String content) {
|
||||
return parse(content, HALL_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseToilet(String content) {
|
||||
return parse(content, TOILET_RESOLVERS)
|
||||
}
|
||||
|
||||
static Object parse(String content, Map<Pattern, Closure> resolvers) {
|
||||
if (content != null) {
|
||||
content = content.replaceAll("\\s", "")
|
||||
}
|
||||
for (def resolver : resolvers.entrySet()) {
|
||||
String match = match(content, resolver.getKey())
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
|
||||
return resolver.getValue()(match)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 成交数据解析工具类
|
||||
*/
|
||||
class DealPriceUtils {
|
||||
|
||||
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
|
||||
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
|
||||
static {
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
|
||||
})
|
||||
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
|
||||
})
|
||||
|
||||
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
|
||||
str = str.replaceAll("室|房", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
|
||||
str = str.replace("厅", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
|
||||
str = str.replace("卫", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
|
||||
str = str.replace("万", '').trim()
|
||||
return (str.toDouble() * 10000).toInteger()
|
||||
})
|
||||
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
|
||||
str = str.replace("元/㎡", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m²"), { String str ->
|
||||
str = str.replace("元/m²", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
|
||||
str = str.replace("元/平", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
|
||||
str = str.replace("元", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
|
||||
str = str.replace("平米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
|
||||
str = str.replace("平方米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
|
||||
str = str.replace("平", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
|
||||
str = str.replace("㎡", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m²"), { String str ->
|
||||
str = str.replace("m²", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
|
||||
str = str.replace("共", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
|
||||
str = str.replace("总", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n2
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n1
|
||||
}
|
||||
return n1
|
||||
})
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
|
||||
str = str.replace("第", "").replaceAll("层|楼", "")
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
|
||||
}
|
||||
/**
|
||||
* 解析成交日期
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Date parseDealDate(String content) {
|
||||
return parse(content, DEAL_DATE_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析室数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseRoom(String content) {
|
||||
return parse(content, ROOM_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHall(String content) {
|
||||
return parse(content, HALL_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseToilet(String content) {
|
||||
return parse(content, TOILET_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析总价
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseTotalPrice(String content) {
|
||||
return parse(content, TOTAL_PRICE_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析单价
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseUnitPrice(String content) {
|
||||
return parse(content, UNIT_PRICE_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析面积
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Double parseArea(String content) {
|
||||
return parse(content, AREA_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析楼层
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseFloor(String content) {
|
||||
return parse(content, FLOOR_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析楼高
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHeight(String content) {
|
||||
return parse(content, HEIGHT_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化小区名
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
static String formatName(String name) {
|
||||
name = name.replace("-商住", "").replace("-别墅", "")
|
||||
//汇福家园平安里(迁)2015-2017
|
||||
name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "")
|
||||
//国贸天悦一期2017
|
||||
name = name.replaceAll("\\d\\d\\d\\d\$", "")
|
||||
if (name ==~ /.+(\(|()[\d一二三四五六七八九十东南西北|\w]+(期|区)(\)|))$/) {
|
||||
name = name.replaceAll("\\(|(|\\)|)", '')
|
||||
} else if (name ==~ /.+(\(|()别墅(\)|))$/) {
|
||||
name = name.replaceAll("\\(|(|\\)|)", '')
|
||||
} else if (name ==~ /.+(\(|().+(\)|))$/) {
|
||||
name = name.replaceAll("(\\(|().+(\\)|))", '')
|
||||
} else if (name.contains("·") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) {
|
||||
println("---------------------------->" + name)
|
||||
name = name.replaceAll("·|-", '')
|
||||
println(name)
|
||||
}
|
||||
if (name ==~ /.{2,}路.{3,}$/ && !(name ==~ /.{2,}路.+号/) && !(name ==~ /.{2,}路.+弄/) && !(name ==~ /.*小区$/)) {
|
||||
name = name.replaceAll("^.{2,}路", "")
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
|
||||
static Object parse(String content, Map<Pattern, Closure> resolvers) {
|
||||
if (content != null) {
|
||||
content = content.replaceAll("\\s", "")
|
||||
}
|
||||
for (def resolver : resolvers.entrySet()) {
|
||||
String match = match(content, resolver.getKey())
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
|
||||
return resolver.getValue()(match)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配符合条件的字符串
|
||||
*
|
||||
* @param str
|
||||
* @param pattern
|
||||
* @return
|
||||
*/
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
static String parsePrice(String priceStr) {
|
||||
def pattern =Pattern.compile("\\d+(\\.\\d+)?万")
|
||||
String match = match(priceStr, pattern)
|
||||
}
|
||||
|
||||
static void main(String[] args) {
|
||||
println parsePrice("顶秀欣园西苑2室1厅1卫492.3万")
|
||||
}
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.Utils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class ZhugeFangDistCrawl {
|
||||
|
||||
public static final SHORT_DATE = Utils.getLastMonth();
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
def task = new ZhugeFangDistCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.manage().deleteAllCookies()
|
||||
getRegions(driver, url)
|
||||
}
|
||||
|
||||
def getRegions(driver, url) {
|
||||
for(def i=1;i<=11;i++){
|
||||
def newUrl = url+"page${i}/"
|
||||
println newUrl
|
||||
def doc = getDocument(newUrl, driver)
|
||||
def total = parseList(doc)
|
||||
|
||||
// def next = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
// if(next){
|
||||
// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def parseList(doc){
|
||||
def city= "福州"
|
||||
def els = JQueryUtils.select("#listTableBox li", doc)
|
||||
if (els) {
|
||||
for(def e : els){
|
||||
def name = e.select(".house-name").text()
|
||||
def str = e.select(".house-adr").text()
|
||||
def address = null
|
||||
if(str){
|
||||
if(str.contains("]")){
|
||||
address = str.substring(str.indexOf("]")+1).trim()
|
||||
}else{
|
||||
address = str
|
||||
}
|
||||
}
|
||||
def regionStr = null
|
||||
if(str && str.contains("]") && str.contains("[")){
|
||||
regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]"))
|
||||
}
|
||||
|
||||
def region = null
|
||||
def block = null
|
||||
if(regionStr){
|
||||
def arrs = regionStr.split("-")
|
||||
if(arrs && arrs.size()>0){
|
||||
if(arrs.size()==2){
|
||||
block = arrs[1].trim()
|
||||
}
|
||||
if(arrs.size()>=1){
|
||||
region = arrs[0].trim()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def links = e.select(".house-name a")
|
||||
def href = null
|
||||
if(links){
|
||||
href = links[0].attr("href")
|
||||
}
|
||||
def priceStr = e.select(".price-number").text()
|
||||
def price = null
|
||||
if(priceStr && !priceStr.contains("暂无")){
|
||||
if(priceStr.contains("元")){
|
||||
priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim()
|
||||
}
|
||||
if(priceStr && priceStr.isNumber()){
|
||||
price = priceStr.toInteger()
|
||||
}
|
||||
}
|
||||
def obj = findByParams(city, region, name)
|
||||
if(obj){
|
||||
if(price){
|
||||
DBObject udata2 = new BasicDBObject()
|
||||
udata2.put("avg_price", price)
|
||||
udata2.put("m_date", new Date())
|
||||
println "[upeate] ${region} "+name
|
||||
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
|
||||
}
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", city)
|
||||
udata.put("region", region)
|
||||
udata.put("block", block)
|
||||
udata.put("url", href)
|
||||
udata.put("address", address)
|
||||
udata.put("name", name)
|
||||
udata.put("avg_price", price)
|
||||
udata.put("s_date", SHORT_DATE)
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("source", "诸葛找房")
|
||||
thirdPartyColl.save(udata)
|
||||
println "[save] ${region} "+name+" "+priceStr
|
||||
}
|
||||
}
|
||||
}
|
||||
return els.size()
|
||||
}
|
||||
|
||||
|
||||
def findByParams(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("source", "诸葛找房")
|
||||
q.put("s_date", SHORT_DATE)
|
||||
return thirdPartyColl.findOne(q);
|
||||
}
|
||||
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
Thread.sleep(10000)
|
||||
def els = JQueryUtils.select("#listTableBox li", doc)
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class DistrictMapMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictMapMatch()
|
||||
task.mapMatch()
|
||||
}
|
||||
|
||||
|
||||
def mapMatch(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d->
|
||||
if(!d['H']){
|
||||
index++
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// if (!map || !map.lat) {
|
||||
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// }
|
||||
if(map){
|
||||
d['F'] = map.id
|
||||
if(map.adname){
|
||||
d['G'] = map.adname
|
||||
}else if(map.region){
|
||||
d['G'] = map.region
|
||||
}
|
||||
d['H'] = map.name
|
||||
d['I'] = map.address
|
||||
if(map.type){
|
||||
d['J'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['J'] = map.estateType
|
||||
}
|
||||
d['K'] = map.lng
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.score
|
||||
d['N'] = map.source
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
}
|
||||
+14
-14
@@ -945,20 +945,20 @@ class CZDataBatch extends BaseBatch{
|
||||
if(price){
|
||||
return price.avg_price;
|
||||
}else{
|
||||
//样例估价
|
||||
def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
|
||||
try{
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
def map = DataHandleUtils.JsonToMap(jsonObject);
|
||||
if(map.code==200){
|
||||
if(map.result){
|
||||
return map.result.avgPrice;
|
||||
}
|
||||
|
||||
}
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
// //样例估价
|
||||
// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
|
||||
// try{
|
||||
// JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
// def map = DataHandleUtils.JsonToMap(jsonObject);
|
||||
// if(map.code==200){
|
||||
// if(map.result){
|
||||
// return map.result.avgPrice;
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }catch(Exception e){
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
//分期找总期
|
||||
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
package com.fangjia.script.temp.shenjian.didata
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
/**
|
||||
* 从excel中匹配出
|
||||
*/
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row ->
|
||||
def city = row['A']
|
||||
def region = row['B']
|
||||
def district_name = row['C']
|
||||
|
||||
def gaodeRegion = row['H']
|
||||
def gaodeName = row['I']
|
||||
|
||||
def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict")
|
||||
if (jsonObject != null && jsonObject.precise != -1) {
|
||||
row['D'] = jsonObject.get("name")
|
||||
}
|
||||
|
||||
if (gaodeRegion && gaodeName) {
|
||||
def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict")
|
||||
if (jsonObject2 != null && jsonObject2.precise != -1) {
|
||||
row['J'] = jsonObject2.get("name")
|
||||
}
|
||||
}
|
||||
});
|
||||
+9
-4
@@ -2,6 +2,8 @@ package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.statistics.poi.AmapPoiDetail
|
||||
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
|
||||
import com.fangjia.script.temp.shenjian.util.ExcelUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
@@ -14,13 +16,15 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class GetPoiFromExcel {
|
||||
|
||||
static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi"
|
||||
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
|
||||
// 住宅类
|
||||
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
|
||||
|
||||
static void main(String[] args) {
|
||||
AtomicLong atomicLong = new AtomicLong(0)
|
||||
//List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
|
||||
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false);
|
||||
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false);
|
||||
for (File file : fileList) {
|
||||
List<List<String>> seaweeds = ExcelUtils.read(file.getPath())
|
||||
for (List<String> infoTemp : seaweeds) {
|
||||
@@ -31,9 +35,10 @@ class GetPoiFromExcel {
|
||||
String city = info.get(0)
|
||||
String region = info.get(1)
|
||||
String seaweedName = info.get(2)
|
||||
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302");
|
||||
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
String jsonStr = JSONObject.toJSONString(pois)
|
||||
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
|
||||
String jsonStr = JSONObject.toJSONString(detailList)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
atomicLong.incrementAndGet()
|
||||
}
|
||||
|
||||
+3
@@ -81,6 +81,9 @@ class SeaweedPoiDetail {
|
||||
if (CollectionUtils.isEmpty(pois)) {
|
||||
pois = GaoDeUtils.searchPoi(city, name, region, typecodes)
|
||||
}
|
||||
if (CollectionUtils.isEmpty(pois)) {
|
||||
pois = GaoDeUtils.searchPoi(city, address, region, typecodes)
|
||||
}
|
||||
|
||||
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
|
||||
// for (AmapPoi amapPoi : pois) {
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger
|
||||
class GaoDeUtils {
|
||||
|
||||
// 新获取key
|
||||
def static GAO_DE_AK = ["4ccfe3c898a9489884bd2bcb1ef6dcde", "4e9592208336433ee1b7aa4d1d434418"]
|
||||
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"]
|
||||
def static AtomicInteger keyUsed = new AtomicInteger(0)
|
||||
|
||||
def static codeNameMap = new HashMap<String, String>(){{
|
||||
|
||||
+2
-1
@@ -30,7 +30,7 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Patter
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 第三方数据标准化处理
|
||||
@@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def filterSpacialChar(def s){
|
||||
def pattern = ~/[()!;\·]\[【】()]/
|
||||
def matcher = pattern.matcher(s)
|
||||
|
||||
def count = matcher.getCount()
|
||||
if(count && count>0){
|
||||
return matcher.replaceAll('_')
|
||||
|
||||
@@ -169,7 +169,7 @@ public class CommonUtils {
|
||||
|
||||
static void sendMessage(String queueName, byte[] bytes){
|
||||
ConnectionFactory factory = new ConnectionFactory();
|
||||
factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f");
|
||||
factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f");
|
||||
Connection conn = factory.newConnection();
|
||||
Channel channel = conn.createChannel();
|
||||
channel.queueDeclare(queueName, true, false, false, null);
|
||||
|
||||
@@ -348,6 +348,34 @@ public class OpenApiClient {
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject standard1Action(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("token", TOKEN);
|
||||
params.put("__refresh_cache","1")
|
||||
|
||||
String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8")
|
||||
println("${city} ${address} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if (result.getInt("code") == 200) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${city} ${address}")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${address} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject standardize(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
|
||||
Reference in New Issue
Block a user