diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d73c64d44..af28b3fd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -58,17 +58,17 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } -// println "curDate:"+curDate -// for(def city : citys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } -// } + println "curDate:"+curDate + for(def city : citys){ +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } + } def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -86,17 +86,16 @@ class BigdataDealBatch extends BaseBatch{ println city +" "+youdaDatas.size() } + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } } @@ -105,7 +104,7 @@ class BigdataDealBatch extends BaseBatch{ fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" } - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成 " + rows.size() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index 9c6b96508..4d8b7fa02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -75,7 +75,7 @@ class FjwPricePkCcbPrice { def cityId = d['D'] def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ - def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202211) + def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209) if(map){ d['O'] = map.avg_price d['P'] = map.calc_type diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 0d15402de..0882eba51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -38,16 +38,16 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] -// CcbCommonUtils.deleteGuidedPrice(curDate) + CcbCommonUtils.deleteGuidedPrice(curDate) for (def city : citys) { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-25", "2022-12-19") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 947d57cb8..b7afc9818 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -47,41 +47,41 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business -def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水'] - -def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -def rets = [] -for(def c : citys){ - def citycode = cityCodeDataMap.get(c) - def datas = findAllData(sqlInstance236, citycode) - def totalCount = datas.size() - def landareaCount = 0; - def buildareaCount = 0 - def greeningrateCount = 0 - def plotratioCount = 0 - for(def m : datas){ - if(m.landarea){ - landareaCount ++ - } - if(m.buildarea){ - buildareaCount++ - } - if(m.greeningrate){ - greeningrateCount++ - } - if(m.plotratio){ - plotratioCount++ - } - } - - def landAreaPre = landareaCount*100/totalCount - def buildAreaPre = buildareaCount*100/totalCount - def greeningratePre = greeningrateCount*100/totalCount - def plotratioPre = plotratioCount*100/totalCount - rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre] -} - -ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率") +//def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水'] +// +//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +//def rets = [] +//for(def c : citys){ +// def citycode = cityCodeDataMap.get(c) +// def datas = findAllData(sqlInstance236, citycode) +// def totalCount = datas.size() +// def landareaCount = 0; +// def buildareaCount = 0 +// def greeningrateCount = 0 +// def plotratioCount = 0 +// for(def m : datas){ +// if(m.landarea){ +// landareaCount ++ +// } +// if(m.buildarea){ +// buildareaCount++ +// } +// if(m.greeningrate){ +// greeningrateCount++ +// } +// if(m.plotratio){ +// plotratioCount++ +// } +// } +// +// def landAreaPre = landareaCount*100/totalCount +// def buildAreaPre = buildareaCount*100/totalCount +// def greeningratePre = greeningrateCount*100/totalCount +// def plotratioPre = plotratioCount*100/totalCount +// rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre] +//} +// +//ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率") def findAllData(sqlInstance236, citycode){ def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'" @@ -1149,6 +1149,30 @@ def findThirdCommCount(cityid, sqlInstance){ +ExcelUtils.modify("D:\\数据\\建行\\二期\\付.xlsx",{d-> + def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district") + if(obj){ + def count = findGuidedPrice(sqlInstance236, obj._id.toString()) + if(count){ + d['M'] = "存在" + } + d['N'] = obj.name_cat + d['O'] = obj.estate_type2 + } + +}) + +def findGuidedPrice(sqlInstance, id){ + def count = 0 + def sql = "select count(*) as count from guided_price_gj where commid='${id}'" + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index ae3aa4800..e87103a90 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -50,7 +50,7 @@ class TradeStatLog { // task.deleteDealData2() // task.statTableCount() // task.filterTrade() - task.updateDealPriceType() +// task.updateDealPriceType() } @@ -90,15 +90,15 @@ class TradeStatLog { } def run(){ - def citys = CcbCommonUtils.citys_92; + def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-08-01" - def date = "2022-09-01" - def endDate = "2022-10-01" - def newDate = "2022-11-01" + def startDate = "2022-09-01" + def date = "2022-10-01" + def endDate = "2022-11-01" + def newDate = "2022-12-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 0231b21c0..47a3b331f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -5,9 +5,36 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import org.apache.commons.io.FileUtils import org.bson.types.ObjectId + +DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + +def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv") +def rets = [] +for(def d : datas){ + def arrs = d.split("@") + rets << ["commId":arrs[0], "avg_price":arrs[4]] +} + +for(def r : rets){ + if(r.avg_price){ + def udata = new BasicDBObject() + udata.put("date", 202210) + udata.put("avg", r.avg_price.toInteger()) + DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false) + }else{ + println r.commId +" 不存在" + } +} + + + + //List list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv")) //println "过滤前:"+list.size()+ " "+new Date().toLocaleString() //def ret = [] @@ -48,29 +75,29 @@ import org.bson.types.ObjectId // return location; //} -List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv")) -def map1 = [:] -for(def s : list1){ - def row = s.split("@\\|@") - def key = row[2]+"##"+row[3]+"##"+row[4] - map1.put(key, row[5]) -} - -List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) -def map2 = [:] -for(def s : list2) { - def row = s.split("@\\|@") - def key = row[2] + "##" + row[3] + "##" + row[4] - map2.put(key, row[5]) -} -def rows = [] -map1.each {k,v-> - def row = [] - def keys = k.split("##") - def lastPrice = map2.get(k) - rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] -} -ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据") +//List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv")) +//def map1 = [:] +//for(def s : list1){ +// def row = s.split("@\\|@") +// def key = row[2]+"##"+row[3]+"##"+row[4] +// map1.put(key, row[5]) +//} +// +//List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) +//def map2 = [:] +//for(def s : list2) { +// def row = s.split("@\\|@") +// def key = row[2] + "##" + row[3] + "##" + row[4] +// map2.put(key, row[5]) +//} +//def rows = [] +//map1.each {k,v-> +// def row = [] +// def keys = k.split("##") +// def lastPrice = map2.get(k) +// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] +//} +//ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据") //ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d-> // def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd")) // if(obj){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index bcbe886de..7e4a0bd2c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202210.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy index d1aca25f7..3075ff1db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy @@ -5,38 +5,112 @@ import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils import com.fangjia.core.util.HttpRequest import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.poi.ss.util.RegionUtil +import java.util.regex.Pattern + class YinLianMeiShi { def baiduCityCodeMap = ["北京": "131", "天津": "332", "石家庄": "150", "唐山": "265", "秦皇岛": "148", "邯郸": "151", "邢台": "266", "保定": "307", "张家口": "264", "承德": "207", "沧州": "149", "廊坊": "191", "衡水": "208", "太原": "176", "大同": "355", "阳泉": "357", "长治": "356", "晋城": "290", "朔州": "237", "晋中": "238", "运城": "328", "忻州": "367", "临汾": "368", "吕梁": "327", "呼和浩特": "321", "包头": "229", "乌海": "123", "赤峰": "297", "通辽": "64", "鄂尔多斯": "283", "呼伦贝尔": "61", "巴彦淖尔": "169", "乌兰察布": "168", "兴安盟": "62", "锡林郭勒盟": "63", "阿拉善盟": "230", "沈阳": "58", "大连": "167", "鞍山": "320", "抚顺": "184", "本溪": "227", "丹东": "282", "锦州": "166", "营口": "281", "阜新": "59", "辽阳": "351", "盘锦": "228", "铁岭": "60", "朝阳": "280", "葫芦岛": "319", "长春": "53", "吉林市": "55", "四平": "56", "辽源": "183", "通化": "165", "白山": "57", "松原": "52", "白城": "51", "延边朝鲜族自治州": "54", "哈尔滨": "48", "齐齐哈尔": "41", "鸡西": "46", "鹤岗": "43", "双鸭山": "45", "大庆": "50", "伊春": "40", "佳木斯": "42", "七台河": "47", "牡丹江": "49", "黑河": "39", "绥化": "44", "大兴安岭地区": "38", "上海": "289", "南京": "315", "无锡": "317", "徐州": "316", "常州": "348", "苏州": "224", "南通": "161", "连云港": "347", "淮安": "162", "盐城": "223", "扬州": "346", "镇江": "160", "泰州": "276", "宿迁": "277", "杭州": "179", "宁波": "180", "温州": "178", "嘉兴": "334", "湖州": "294", "绍兴": "293", "金华": "333", "衢州": "243", "舟山": "245", "台州": "244", "丽水": "292", "合肥": "127", "芜湖": "129", "蚌埠": "126", "淮南": "250", "马鞍山": "358", "淮北": "253", "铜陵": "337", "安庆": "130", "黄山": "252", "滁州": "189", "阜阳": "128", "宿州": "370", "巢湖": "251", "六安": "298", "亳州": "188", "池州": "299", "宣城": "190", "福州": "300", "厦门": "194", "莆田": "195", "三明": "254", "泉州": "134", "漳州": "255", "南平": "133", "龙岩": "193", "宁德": "192", "南昌": "163", "景德镇": "225", "萍乡": "350", "九江": "349", "新余": "164", "鹰潭": "279", "赣州": "365", "吉安": "318", "宜春": "278", "抚州": "226", "上饶": "364", "济南": "288", "青岛": "236", "淄博": "354", "枣庄": "172", "东营": "174", "烟台": "326", "潍坊": "287", "济宁": "286", "泰安": "325", "威海": "175", "日照": "173", "莱芜": "124", "临沂": "234", "德州": "372", "聊城": "366", "滨州": "235", "菏泽": "353", "郑州": "268", "开封": "210", "洛阳": "153", "平顶山": "213", "安阳": "267", "鹤壁": "215", "新乡": "152", "焦作": "211", "濮阳": "209", "许昌": "155", "漯河": "344", "三门峡": "212", "南阳": "309", "商丘": "154", "信阳": "214", "周口": "308", "驻马店": "269", "武汉": "218", "黄石": "311", "十堰": "216", "宜昌": "270", "襄阳": "156", "鄂州": "122", "荆门": "217", "孝感": "310", "荆州": "157", "黄冈": "271", "咸宁": "362", "随州": "371", "恩施土家族苗族自治州": "373", "仙桃": "1713", "潜江": "1293", "天门": "2654", "神农架林区": "2734", "长沙": "158", "株洲": "222", "湘潭": "313", "衡阳": "159", "邵阳": "273", "岳阳": "220", "常德": "219", "张家界": "312", "益阳": "272", "郴州": "275", "永州": "314", "怀化": "363", "娄底": "221", "湘西土家族苗族自治州": "274", "广州": "257", "韶关": "137", "深圳": "340", "珠海": "140", "汕头": "303", "佛山": "138", "江门": "302", "湛江": "198", "茂名": "139", "肇庆": "338", "惠州": "301", "梅州": "141", "汕尾": "339", "河源": "200", "阳江": "199", "清远": "197", "东莞": "119", "中山": "187", "潮州": "201", "揭阳": "259", "云浮": "258", "南宁": "261", "柳州": "305", "桂林": "142", "梧州": "304", "北海": "295", "防城港": "204", "钦州": "145", "贵港": "341", "玉林": "361", "百色": "203", "贺州": "260", "河池": "143", "来宾": "202", "崇左": "144", "海口": "125", "三亚": "121", "五指山": "1644", "琼海": "2358", "儋州": "1215", "文昌": "2758", "万宁": "1216", "东方": "2634", "定安": "1214", "屯昌": "1641", "澄迈": "2757", "临高": "2033", "白沙黎族自治": "2359", "昌江黎族自治": "1642", "乐东黎族自治": "2032", "陵水黎族自治": "1643", "保亭黎族苗族自治": "1217", "琼中黎族苗族自治": "2031", "重庆": "132", "成都": "75", "自贡": "78", "攀枝花": "81", "泸州": "331", "德阳": "74", "绵阳": "240", "广元": "329", "遂宁": "330", "内江": "248", "乐山": "79", "南充": "291", "眉山": "77", "宜宾": "186", "广安": "241", "达州": "369", "雅安": "76", "巴中": "239", "资阳": "242", "阿坝藏族羌族自治州": "185", "甘孜藏族自治州": "73", "凉山彝族自治州": "80", "贵阳": "146", "六盘水": "147", "遵义": "262", "安顺": "263", "铜仁地区": "205", "黔西南布依族苗族自治州": "343", "毕节地区": "206", "黔东南苗族侗族自治州": "342", "黔南布依族苗族自治州": "306", "昆明": "104", "曲靖": "249", "玉溪": "106", "保山": "112", "昭通": "336", "丽江": "114", "临沧": "110", "楚雄彝族自治州": "105", "红河哈尼族彝族自治州": "107", "文山壮族苗族自治州": "177", "普洱": "108", "西双版纳傣族自治州": "109", "大理白族自治州": "111", "德宏傣族景颇族自治州": "116", "怒江傈僳族自治州": "113", "迪庆藏族自治州": "115", "拉萨": "100", "昌都地区": "99", "山南地区": "97", "日喀则地区": "102", "那曲地区": "101", "阿里地区": "103", "林芝地区": "98", "西安": "233", "铜川": "232", "宝鸡": "171", "咸阳": "323", "渭南": "170", "延安": "284", "汉中": "352", "榆林": "231", "安康": "324", "商洛": "285", "兰州": "36", "嘉峪关": "33", "金昌": "34", "白银": "35", "天水": "196", "武威": "118", "张掖": "117", "平凉": "359", "酒泉": "37", "庆阳": "135", "定西": "136", "陇南": "256", "临夏回族自治州": "182", "甘南藏族自治州": "247", "西宁": "66", "海东地区": "69", "海北藏族自治州": "67", "黄南藏族自治州": "70", "海南藏族自治州": "68", "果洛藏族自治州": "72", "玉树藏族自治州": "71", "海西蒙古族藏族自治州": "65", "银川": "360", "石嘴山": "335", "吴忠": "322", "固原": "246", "中卫": "181", "乌鲁木齐": "92", "克拉玛依": "95", "吐鲁番地区": "89", "哈密地区": "91", "昌吉回族自治州": "93", "博尔塔拉蒙古自治州": "88", "巴音郭楞蒙古自治州": "86", "阿克苏地区": "85", "克孜勒苏柯尔克孜自治州": "84", "喀什地区": "83", "和田地区": "82", "伊犁": "90", "塔城地区": "94", "阿勒泰地区": "96", "石河子": "770", "阿拉尔": "731", "图木舒克": "792", "五家渠": "789", "香港特别行政区": "2912", "澳门特别行政区": "2911"] + DBCollection QCC_COLL = MongoUtils.getCollectionByName("crawl", "tyc_web_crawl", "192.168.0.206",27020) + def filters = [] static main(args) { def task = new YinLianMeiShi(); - task.run() +// task.run() +// task.matchFlag() +// task.marge() + task.addValueByBaiduId() + } + + def addValueByBaiduId(){ + ExcelUtils.modify("D:\\数据\\银联\\数据500条(12-21).xlsx",{d-> + if(d['A'] && d['T']=="百度人工"){ + def jsonObj = BaiDuApiUtils.getDetailByUid(d['U']) + if(jsonObj && jsonObj.result){ + def result = jsonObj.result + if(result){ + d['J'] = result.uid + d['K'] = RegionData.formatRegion(result.city, result.area) + d['L'] = result.name + d['M'] = result.address + d['N'] = result.detail_info.tag + d['O'] = result.location.lng + d['P'] = result.location.lat + println result + } + } + } + }) + } + def marge(){ + def file1 = "D:\\数据\\银联\\银联人工1221(灵芝).xlsx" + def datas1 = ExcelUtils.read(file1, 0, 1) + ExcelUtils.modify("D:\\数据\\银联\\数据500条(12-21).xlsx",{d-> + if(d['A']){ + for(def r : datas1){ + if(r['A'] == d['A']){ + List pois = GaoDeUtils.getAmapById2(r['F']); + if(pois && pois.size()==1){ + def amapPoi = pois[0] + d['J'] = amapPoi.id + d['K'] = amapPoi.adname + d['L'] = amapPoi.name + d['M'] = amapPoi.address + d['N'] = amapPoi.type + def location = amapPoi.location + if(location){ + def arrs = location.split(",") + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + d['O'] = lng2 + d['P'] = lat2 + } + } + println pois + } + } + } + }) } def run(){ - ExcelUtils.modify("D:\\数据\\银联\\数据500条-关键字.xlsx",{row-> + ExcelUtils.modify("D:\\数据\\银联\\数据500条(合并人工).xlsx",{row-> def score = row['Q'] - if(row['E'] && (!score || (score && score.toString().toDouble()<0.8))){ + if(row['B'] && !row['L']){ + def baiduCode = getBaiduMapCode(row['E']); def city = RegionData.formatCity(row['E']); + if(!baiduCode){ + baiduCode = getBaiduMapCode(city); + } - def baiduCode = getBaiduMapCode(city); def newKeys = filterName(row['D'], row['E'], row['F'], row['B']) row['I'] = newKeys if(baiduCode){ - def rets = searchBaiDuMap(row['B'], baiduCode.toInteger()); - def simpleAddr = getSimpleAddress(city, row['C']) + def map = getSimpleAddress(city, row['C']) + def simpleAddr = map.addr + if(map.city!="市辖区" && map.city!=city){ + city = RegionData.formatCity(map.city); + } + def rets = getBiduSearchDatas(baiduCode , simpleAddr, row['B'], null) + rets = getBiduSearchDatas(baiduCode , null, row['B'], rets) + rets = getBiduSearchDatas(baiduCode , null, simpleAddr, rets) def bdRes = searchMapBySimilar(city, row['F'], row['B'], simpleAddr, rets, newKeys); - if (bdRes && (!score || (score && bdRes.score>score.toString().toDouble()))) { + if (bdRes) { row['J'] = bdRes.uid row['K'] = bdRes.region row['L'] = bdRes.name @@ -56,29 +130,155 @@ class YinLianMeiShi { }) } + def matchFlag(){ + ExcelUtils.multiThreadingModify("D:\\数据\\银联\\数据500条(合并人工).xlsx",{row-> + if(row['B'] && row['B'].contains("麦加美董村餐厅")){ + def datas = findQccData(row['B']) + if(datas){ + def max_score = 0 + def last_obj = null + def newKeys = filterName(row['D'], row['E'], row['F'], row['B']) + def city = RegionData.formatCity(row['E']); + for(def d : datas){ + + def map = getSimpleAddress(null, d.address) + def inputMap = getSimpleAddress(null, row['C']) + if(inputMap && inputMap.city){ + city = inputMap.city + } + if(map && city==map.city){ + def qccNameKey = filterName(map.pro, map.city, map.region, d.name) + def name_score = CosineSimilarAlgorithm.getSimilarityRatio(newKeys, qccNameKey); + def addr_score = 0 + if (map.addr && inputMap.addr) { + if(map.addr.contains("与")){ + def arrs = map.addr.split("与") + for(def s1 : arrs){ + def temp_addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inputMap.addr, s1); + if(temp_addr_score > addr_score){ + addr_score = temp_addr_score + } + } + }else{ + addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inputMap.addr, map.addr); + } + } + + def total_score = name_score + addr_score + if(total_score > max_score){ + max_score = total_score; + last_obj = d + last_obj.simple_addr = map.addr + last_obj.tyc_city = map.city + } + println row['B'] +" "+ d.name+" "+total_score + }else{ + def qccNameKey = filterName(map.pro, map.city, map.region, d.name) + def name_score = CosineSimilarAlgorithm.getSimilarityRatio(newKeys, qccNameKey); + if(map.addr && inputMap.addr && (inputMap.addr.contains(map.addr) || map.addr.contains(inputMap.addr))){ + max_score = name_score; + last_obj = d + last_obj.simple_addr = map.addr + last_obj.tyc_city = map.city + println row['B'] +" "+ d.name+" "+name_score + } + } + } + if(max_score && max_score > 0.4){ + row['AB'] = last_obj.name + row['AC'] = last_obj.address + row['AD'] = last_obj.flag + row['AE'] = max_score + } + } + } + }); + } + + def findQccData(keywords){ + def q = new BasicDBObject() +// q.put("search_terms", keywords) + if(keywords.contains("(*")){ + keywords = keywords.toString().substring(0, keywords.toString().indexOf("(*")) + } + q.put("search_terms", Pattern.compile("^.*${keywords}.*\$", Pattern.CASE_INSENSITIVE)) + def dbcursor = QCC_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << ["name":d.name, "address":d.taxAddress, "flag":d.regStatus] + } + return rets; + } + def getBiduSearchDatas(baiduCode , simpleAddr, nameKey, rets){ + def searchKey = "" + if(simpleAddr){ + searchKey = simpleAddr +" "+nameKey + }else{ + searchKey = nameKey + } + def datas = searchBaiDuMap(searchKey, baiduCode.toInteger()); + if(!rets){ + return datas; + }else{ + def ids = [] + for(def r : rets){ + if(!ids.contains(r.uid)){ + ids << r.uid + } + } + for(def d : datas){ + if(!ids.contains(d.uid)){ + rets << d + } + } + return rets; + } + return null; + } + def getSimpleAddress(city, address){ def jsonObj = FangjiaClientUtils.addressStructure(city, address) - def old_addr = null + def old_addr = "" + def map = [:] if(jsonObj && jsonObj.result){ + + if(jsonObj.result.province){ + map.pro = jsonObj.result.province + } + + if(jsonObj.result.city){ + if(jsonObj.result.city=="市辖区"){ + map.city = jsonObj.result.province + }else{ + map.city = jsonObj.result.city + } + map.city = RegionData.formatCity(map.city) + } + + if(jsonObj.result.district){ + map.region = jsonObj.result.district + } if(jsonObj.result.street){ def street = jsonObj.result.street if(street){ old_addr = street + map.street = street.replace("交叉口","").replace("交叉","") + } + } + if(jsonObj.result.streetNumber){ + def street_number = jsonObj.result.streetNumber + if(street_number){ + old_addr = old_addr + street_number } } -// if(jsonObj.result.streetNumber){ -// def street_number = jsonObj.result.streetNumber -// if(street_number){ -// old_addr = old_addr + street_number -// } -// } } - return old_addr; + map.addr = old_addr.replace("交叉口","").replace("交叉","") + return map; } def filterName(pro,city, region, name){ - def newKey = name.replace("有限责任公司", "").replace("有限公司","") + def newKey = name.replace("餐饮服务有限公司","").replace("控股有限公司","").replace("有限责任公司", "").replace("有限公司","").replace("餐饮","") if(pro && name.contains(pro)){ newKey = newKey.replace(pro, "") } @@ -88,17 +288,57 @@ class YinLianMeiShi { newKey = newKey.replace(city, "").replace(newCity, "") } - - if(region && newKey.contains(region)){ + if(region && newKey.contains(region) ){ newKey = newKey.replace(region, "") } - if(newKey.contains("(")){ - def subName = newKey.substring(newKey.indexOf("(")+1, newKey.indexOf(")")); - if(subName.endsWith("街") || subName.endsWith("路") || subName.contains("*")){ - newKey = newKey.replace("("+subName+")", ""); - } - newKey = newKey.replace("(","").replace(")","") + + def newRegion = null + if(region){ + newRegion = RegionData.formatRegion(city, region) } + if(newRegion && newKey.contains(newRegion)){ + newKey = newKey.replace(newRegion, "") + } + + if(newKey.contains("(") || newKey.contains("(")){ + if(newKey.contains("(")){ + def subName = newKey.substring(newKey.indexOf("(")+1, newKey.indexOf(")")); + if(subName.endsWith("街") || subName.endsWith("路") || subName.contains("*") || subName.endsWith("店")){ + newKey = newKey.replace("("+subName+")", ""); + } + } + newKey = newKey.replace("(","").replace(")","").replace("(","").replace(")","") + if(newKey.toString().endsWith("店")){ + newKey = newKey.substring(0, newKey.size()-1) + } + } + return newKey; + } + + def filterName2(pro,city, region, name){ + def newKey = name.replace("餐饮服务有限公司","").replace("控股有限公司","").replace("有限责任公司", "").replace("有限公司","").replace("餐饮","") + if(pro && name.contains(pro)){ + newKey = newKey.replace(pro, "") + } + + def newCity = RegionData.formatCity(city) + if(city && (name.contains(city) || name.contains(newCity))){ + newKey = newKey.replace(city, "").replace(newCity, "") + } + + if(region && newKey.contains(region) ){ + newKey = newKey.replace(region, "") + } + + def newRegion = null + if(region){ + newRegion = RegionData.formatRegion(city, region) + } + if(newRegion && newKey.contains(newRegion)){ + newKey = newKey.replace(newRegion, "") + } + + newKey = newKey.replace("(","").replace(")","").replace("(","").replace(")","") return newKey; } @@ -113,7 +353,8 @@ class YinLianMeiShi { def rets = [] try { Thread.sleep(500) - String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&auth=AUeHbGFG3ZMg2Bv8Y2b8GfVB3OGc35VOuxLNVzzzLHHtSAFSAS7I7vICu2I%400GdFvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkvyuw904v7uv%40vcuVtvc3CuVtvcPPuVtveGvuNzHtAimNzCywiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3%40SdJMjjejCEBf0wd0vyMO7Ay7MOI&seckey=Im7p%2FanwNmQx078ijew%2Fsx5QUzMQU8M7SbrqWr3JGEc%3D%2C0-Ulo82T3hd3wuKMf_tTM6DbOSuS4Zlg9ZEtPAqKkNKXysGCBLK7zryshf7oP6p8_Hynb5vDXmd6C0TtcF9impZZK2kzkn1_SWeBVEJH4Zdh0n81wXv-1j6enHaAwiYj7T7eCaCw3g5vDcHvZxUxtBsRphpZI-Rco19dgpXwT9xb-XVcCb8GyvbMt5s2wIpb"; + // String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&&auth=Dae1RaNR1LK8xdJTxN8SQOTRa2AH%3DL7vuxLNVENTxRBt6BG6B6WJWwGvzBU%40yYxvkGcuVtvvhguVtvyheuVtcvY1SGpuLtB2QKAZzvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwBVDzvFosSE21wiKDv7uvhgMuzVVtvrMhuENLtSrrlrIKHf0wd0vyMO7FOUySy&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a"; + String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=after_baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=19&from=webmap&sug_forward=&auth=E8LCawOwRbEOx6JVTvCIBI6W7zSz5vSXuxLNVENTRVLtUDHUDUYKYyHxzBU%40yYxvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkv7uvZgMuVtv%40vcuVtvc3CuVtvcPPuztxgHxwDWBWvveh3uxtwiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3VP8JSrrlrIKHf0wd0vyMO7FOUS7F&seckey=Im7p%2FanwNmQx078ijew%2FsyQrDC7YJnynqrZ43JVFlM0%3D%2C0-Ulo82T3hd3wuKMf_tTM-ZXYa3S1Y21RX3DV_l5MHgI_1U55ZQrzxTIKi3BRb2x59mVOurDjnW_8CwMEmjK3y8JfHyV0kv70GMQXxnIEcHpM3Dqhs5EdrN6ciDtoxyrcRfowp_xTBkEgU49ZrCz2KeLyujCOypqjKS3rAmPOfEp1kIZPD4W6RDQzrsWTe7a&device_ratio=2&tn=B_NORMAL_MAP&nn=0&u_loc=13513133,3631861&ie=utf-8&t=${System.currentTimeMillis()}&newfrom=zhuzhan_webmap" def content = new HttpRequest().get(url, null, "utf-8") JSONObject json = JSONObject.fromObject(content) if (json && json.content && json.content.size() > 0) { @@ -143,14 +384,21 @@ class YinLianMeiShi { bean.longitude = lng bean.latitude = lat bean.name = loc.name - bean.address = getAddressInfo(loc.address_norm, loc.addr); + if(loc.address_norm){ + bean.address = getAddressInfo(loc.address_norm, loc.addr); + } bean.old_addr = loc.addr -// def baiduCity = RegionData.formatCity(loc.city_name); + if(!baiduCity){ + baiduCity = RegionData.formatCity(loc.city_name) + } bean.cityname = baiduCity - def regionStr = loc.area_name.replace(loc.city_name, "") - def baiduRegion = RegionData.formatRegion(baiduCity, regionStr); - bean.region = baiduRegion + if(loc.city_name){ + def regionStr = loc.area_name.replace(loc.city_name, "") + def baiduRegion = RegionData.formatRegion(baiduCity, regionStr); + bean.region = baiduRegion + } + if(loc.std_tag){ bean.type = loc.std_tag } @@ -161,6 +409,7 @@ class YinLianMeiShi { } } } catch(Exception e) { + searchBaiDuMap(address, cityCode) } return rets; } @@ -226,10 +475,6 @@ class YinLianMeiShi { def maxScore = 0; if(!bo || !maxScore){ for(def o :rets){ - - if(!region.contains(o.region)){ - continue; - } def s = o.name // if(o.name && o.name.contains("(") && !o.name.toString().startsWith("(")){ // s = o.name.substring(0, o.name.indexOf("(")) @@ -237,29 +482,76 @@ class YinLianMeiShi { // s = formatName(s) def addr = o.address def mapKey = filterName(null, o.cityname, o.region, o.name) + def mapKey1 = filterName2(null, o.cityname, o.region, o.name) if(mapKey){ def score = CosineSimilarAlgorithm.getSimilarityRatio(mapKey, newKey); - if(city==o.cityname){ - if(score > maxScore){ - maxScore = score; - o.matchType = "城市相同,名称相似" - o.city = o.cityname - bo = o - bo.score = maxScore + def score1 = CosineSimilarAlgorithm.getSimilarityRatio(mapKey1, newKey); + if(score1>score){ + score = score1 + } + def addr_score = 0 + if(inpAddr) { + def baidu_addr_map = getSimpleAddress(o.cityname, o.old_addr) + if(!baidu_addr_map.addr){ + baidu_addr_map.addr = AddressUtil.formatAddress(o.cityname, o.old_addr) } - if(bo && score!=1){ - if(inpAddr){ - def baidu_addr = getSimpleAddress(o.cityname, o.old_addr) - if(baidu_addr){ - def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr); - if(addr_score==1){ - o.matchType = "城市相同,名称相似,地址有关" - bo.score = bo.score + 0.1 + if(!baidu_addr_map.addr){ + if(o.region && !region.contains(o.region)){ + continue; + } + if(city!=o.cityname){ + continue; + } + } + if (baidu_addr_map.addr) { + if(baidu_addr_map.addr.contains("与")){ + def arrs = baidu_addr_map.addr.split("与") + for(def s1 : arrs){ + def temp_addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, s1); + if(temp_addr_score > addr_score && (inpAddr.contains(baidu_addr_map.addr) || baidu_addr_map.addr.contains(inpAddr))){ + addr_score = temp_addr_score } } + }else{ + addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr_map.addr); + if(addr_score>0 && inpAddr && baidu_addr_map.street && !inpAddr.contains(baidu_addr_map.street) && !baidu_addr_map.street.contains(inpAddr)){ + addr_score = 0 + } } + } + } + + if(addr_score>0 && score==0){ + addr_score = addr_score/2 + } + def total_score = score + addr_score +// if(city==o.cityname){ + if(total_score > maxScore){ + maxScore = total_score; + o.matchType = "城市相同,名称相似" + o.city = o.cityname + bo = o + bo.score = maxScore + + if(addr_score>0.6){ + o.matchType = "城市相同,名称相似,地址有关" + } + } +// if(bo && score!=1){ +// if(inpAddr){ +// def baidu_addr = getSimpleAddress(o.cityname, o.old_addr) +// if(baidu_addr){ +// def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr); +// //if(addr_score==1){ +// // o.matchType = "城市相同,名称相似,地址有关" +// bo.score = bo.score + addr_score +// maxScore = bo.score; +// //} +// } +// } + // for(def word : filters){ // if(mapKeyName.toString().endsWith(word)){ // mapKeyName = mapKeyName.replace(word, "") @@ -292,8 +584,8 @@ class YinLianMeiShi { // } // } // } - } - } +// } +// } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 6d692218b..4f5efcea8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -52,7 +52,7 @@ class YouDaDataParser { static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2022年12月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0ecdddeb1..7f99588f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia")){ + if (!href.contains("fang.lianjia") ){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 17b02bb6b..2a34dd4b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -247,6 +247,35 @@ class GaoDeUtils { return Collections.emptyList() } + + static List getAmapById2(id){ + def params = [:] + params.id = id + params.output = 'json' + params.key = "66e6771767ab293ace441553c36621c1" + JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params); + } + if (json && json.info && json.info == "OK") { + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + return ret + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return Collections.emptyList() + } + /** * 获取详情 */ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 78d3fceac..0ceed1225 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "链家"// - def citys = findCrawlCity(source, 202209); + def source = "365淘房"// + def citys = findCrawlCity(source, 202210); for(def city : citys){ - def count1 = findCount(source, city, 202209); - def count2 = findCount(source, city, 202210); + def count1 = findCount(source, city, 202210); + def count2 = findCount(source, city, 202211); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy index 34186b830..ec6fa0a0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy @@ -65,6 +65,8 @@ class MongoUtils { if (domainUrl == MongoDB_Domain_mongo) { port == 28017; } + + if ((domainUrl == "192.168.0.136") && (port == 27017 || port == 28017)) { return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name) }