diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 6a9d4446e..9af443b40 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -225,17 +225,42 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202202) +//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01") +//def testMap = findCaluateCityPPE(sqlInstance236, 202203) //ExcelUtils.modify(fileName,{d-> // if(d['C']){ // def oldcount = dealMap.get(d['C']) -// d['V'] = oldcount +// d['X'] = oldcount // def casecount = testMap.get(d['C']) -// d['W'] = casecount +// d['Y'] = casecount // println d // } //},0) +def rows = findTradePpeData(sqlInstance236) +// rows.unique() +def map=findTradeDetail(sqlInstance236,null) +def datas=[] +map.each{key,row-> + if(!rows.contains(key)){ + datas< +// def rows = findTradePpeData(sqlInstance236, 202203,code) +// def alltrades=findTradeDetail(sqlInstance236,code) +// alltrades.each{it-> +// def commid=it[0] +// if(!rows.contains(commid)){ +// datas< // if(d['I'] && d['K']){ @@ -270,24 +295,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // } //},0) -def index = 0 -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> - def flag = d['M'] - if(flag==0.0){ - if(d['D']){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", new ObjectId(d['D'])) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - index++ - println index - } - } - -},0) +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> +// def flag = d['M'] +// if(flag==0.0){ +// if(d['D']){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", new ObjectId(d['D'])) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// index++ +// println index +// } +// } +// +//},0) //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> // if(d['A']){ @@ -558,8 +583,30 @@ def findCaluateCityPPE(def sqlInstance, def yearMonth){ } - - +def findTradePpeData(def sqlInstance){ + def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'"; + def rows = new HashSet() + sqlInstance.eachRow(sql,{r-> + rows< + def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate + def row=[] + row< + if(row['N'] && row['N'].toString().toDouble()>=20){ + println row['F'] +" "+row['N'] + def dist = CommonDatautils.findDistrictById(row['C']) + for(def weekend : weekends){ + CommonUtils.recalculationAverage(row['D'], row['E'], row['F'], dist.estate_type2, weekend,false,0) + } +// FangjiaClientUtils.sendDistrictPriceCalculate(row['D'], row['E'], row['F'], 48); + } + },0) + } + + def run1(){ + def city = "重庆" + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V2.xlsx", { row -> + def region = row['E'] + def name = row['F'] + def id = row['C'] + def lastWeekend = DateUtils.getFirstDayOfWeek(new Date()-30) + def lastYearWeekend = DateUtils.getFirstDayOfWeek(new Date()-365) + if (region && name) { + row['H'] = CommonDatautils.findNewSqmetreRent(city, region, name) + + def priceObj = CommonDatautils.findDistrictPrice(city, region, name) + if(priceObj){ + row['AB'] = priceObj.avg_price + } + + def dist = CommonDatautils.findDistrictById(id) + if(dist) { + List list = DbUtils.findAroundDistrict(dist) + def map = getArroundAvgPriceMap(list) + if(map){ + row['L'] = map.avg_price + row['Q'] = map.max_val + row['R'] = map.min_val + } + def lastArroundPrice = getArroundAvgPriceByWeekend(list, lastWeekend) + if(lastArroundPrice && priceObj){ + def mod = (map.avg_price-lastArroundPrice)*100/lastArroundPrice + mod = DataHandleUtils.getDoubleRound(2, mod) + row['P'] = mod + } + def lastYearArroundPrice = getArroundAvgPriceByWeekend(list, lastYearWeekend) + if(lastYearArroundPrice && priceObj){ + def mod = (map.avg_price-lastYearArroundPrice)*100/lastYearArroundPrice + mod = DataHandleUtils.getDoubleRound(2, mod) + row['O'] = mod + } + } + def lastWeekendObj = CommonDatautils.queryDistrictPrice(city, region, name, lastWeekend) + if(lastWeekendObj){ + row['S'] = lastWeekendObj.avg_price + if(priceObj){ + def mod = (priceObj.avg_price-lastWeekendObj.avg_price)*100/lastWeekendObj.avg_price + mod = DataHandleUtils.getDoubleRound(2, mod) + row['M'] = mod + } + } + def lastYearObj = CommonDatautils.queryDistrictPrice(city, region, name, lastYearWeekend) + if(lastYearObj){ + row['T'] = lastYearObj.avg_price + if(priceObj){ + def mod = (priceObj.avg_price-lastYearObj.avg_price)*100/lastYearObj.avg_price + mod = DataHandleUtils.getDoubleRound(2, mod) + row['N'] = mod + } + } + + } + },0) + } + + + def run(){ + def city = "重庆" + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆估值交易信息V2.xlsx", { row -> + def region = row['E'] + def name = row['F'] + def id = row['C'] + def area = row['G'].toDouble() + if(region && name){ + def priceObj = CommonDatautils.findDistrictPrice(city, region, name) + if(priceObj){ + row['J'] = priceObj.avg_price + if(row['C']){ + row['I'] = (area*priceObj.avg_price).toInteger() + } + } + def dist = CommonDatautils.findDistrictById(id) + if(dist){ + List list = DbUtils.findAroundDistrict(dist) + row['N'] = getArroundAvgPrice(list) + + def alias = dist.alias + if(!alias){ + alias = [] + } + alias << dist.name + def houseAvgPrice = findSimAreaHousePrice(dist.city, dist.region, alias,20220101,20220401, area-10, area+10) + if(houseAvgPrice){ + row['K'] = houseAvgPrice + } + + def tradeAvgPrice = findTrades(dist, DateUtils.parseFromTightYYYYMMDD("2021-12-01"), area-10, area+10) + row['L'] = tradeAvgPrice + + def avg_price = findAvgPrice(dist._id.toString()) + if(row['J'] && avg_price){ + def weekendPrice = row['J'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + row['P'] = ppe15 + } + } + } + }); + } + + def findTrades(dist, startDate, startArea, endArea){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + // q.put("source", "链家在线") + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + + q.put("trade_date", new BasicDBObject('$gte', startDate)) + q.put("area", new BasicDBObject('$gte', startArea).append('$lt',endArea)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def prices = [] + for(def d : dbcorsor){ + if(d.avg_price){ + prices << d.avg_price + } + } + + def price = null + if(prices){ + price = (prices.sum()/prices.size()).toInteger() + } + return price + } + + + def getArroundAvgPriceMap(def list){ + def prices = [] + for(def l : list){ + def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + prices.sort() + def map = [:] + map.min_val = prices[0] + map.max_val = prices[prices.size()-1] + def avg_price = (prices.sum()/prices.size()).toInteger() + map.avg_price = avg_price + return map; + } + + def getArroundAvgPrice(def list){ + def prices = [] + for(def l : list){ + def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + def avg_price = (prices.sum()/prices.size()).toInteger() + return avg_price; + } + + + def getArroundAvgPriceByWeekend(def list, def weekend){ + def prices = [] + for(def l : list){ + def priceObj =CommonDatautils.queryDistrictPrice(l.city, l.region, l.name, weekend) + if(priceObj){ + prices << priceObj.avg_price + } + } + def avg_price = null; + if(prices){ + avg_price = (prices.sum()/prices.size()).toInteger() + } + return avg_price; + } + + + def findSimAreaHousePrice(def city, def region, def alias, def startDate, def endDate, def startArea, def endArea){ + def aliasStr = null; + def total = 0 + def prices = [] + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and area>=${startArea} and area<=${endArea}" + + ElasticUtils.executeSqlQuery(sql,{ data-> + prices << data.avg_price + }) + }catch(Exception e){ + e.printStackTrace() + } + + def price = null + if(prices){ + price = (prices.sum()/prices.size()).toInteger() + } + return price + } + + public String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy new file mode 100644 index 000000000..75c731a7f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy @@ -0,0 +1,84 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class BlankMatch { + static main(args) { + def task = new BlankMatch() +// task.run() +// task.run1() +// task.run2() + task.run3() + } + + def run3(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district") + if(dist){ + row['C'] = dist._id.toString() + } + } + },0) + } + def run2(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1) + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + for(def d : datas){ + if(d['B'] == row['B']){ + row['C'] = d['C'] + row['D'] = d['D'] + row['E'] = d['E'] + row['F'] = d['J'] + break; + } + } + } + },0) + } + + def run() { + //rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); + ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['C'] = jsonObject.get("id") + row['D'] = jsonObject.get("city") + row['E'] = jsonObject.get("district") + row['F'] = jsonObject.get("name") + row['G'] = jsonObject.get("address") + } + } + + + }); + } + + def run1() { + //rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx", { row -> + if(row['F']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['F'], "property", false, row['E'], null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['C'] = jsonObject.get("id") + row['D'] = jsonObject.get("city") + row['J'] = jsonObject.get("name") + } + + } +// def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) +// if(jsonObject != null && jsonObject.precise!=-1) { +// row['C'] = jsonObject.get("id") +// row['D'] = jsonObject.get("city") +// row['E'] = jsonObject.get("district") +// row['F'] = jsonObject.get("name") +// row['G'] = jsonObject.get("address") +// } + }); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FieldAddValue.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FieldAddValue.groovy new file mode 100644 index 000000000..01ca3f3be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/FieldAddValue.groovy @@ -0,0 +1,77 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import org.apache.lucene.queryparser.classic.QueryParser +import org.bson.types.ObjectId + +class FieldAddValue { + static main(args) { + def task = new FieldAddValue() + task.run() + } + + def run(){ + //16991 14360 + def districts = CommonDatautils.findAllDistrictByCity("重庆", "district", ["region","name","alias","is_elevator"]) + def index = 0 + def updateIndex = 0 + for(def d : districts){ + if(!d.is_elevator){ + index++ + println index + def alias = d.alias + if(!d.alias){ + alias = [] + } + alias << d.name + def maxHeight = findHouseHeight("重庆", d.region, alias, 20220101) + if(maxHeight && maxHeight>6){ + updateIndex++ + def udata = new BasicDBObject() + udata.put("is_elevator", true) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId( d._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + } + } + println index+" "+updateIndex + } + + def findHouseHeight(def city, def region, def alias, def startDate){ + def aliasStr = null; + def total = 0 + def heights = null + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select max(height) as height from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name in ('${aliasStr}') " + + ElasticUtils.executeSqlQuery(sql,{ data-> + if(data.height){ + heights = data.height + } + + }) + }catch(Exception e){ + e.printStackTrace() + } + +// def max_height = null +// if(heights){ +// heights.sort() +// max_height = heights[heights.size()-1] +// } + return heights + } + + public String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index ba7287128..244ae41c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils class OpenApiUtils { static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ - address = address.replace("(","(").replace(")",")").replace(" ","") + address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" if(preciseLevelflag){ url = url +"&preciseLevel=strict" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy index b43da6c82..df875945a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy @@ -48,7 +48,7 @@ class VagueAddressLocationBatch extends BaseBatch { void process(Object args) { //"北京","常州", // def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity(); - def citys =["北京"] //CommonDatautils.findAllCity(); + def citys =["重庆"] //CommonDatautils.findAllCity(); //,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄" //市谊大厦 华三新村 阳光家园 for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy index 92e8f1555..7730b81cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy @@ -55,7 +55,7 @@ class AddDistrictFieldValue { // } // }) -// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys; + def citys = ["重庆"]//CcbCommonUtils.citys; //第三方均价数据匹配 // for(def city : citys){ // def rets = findAllDataByCity(city) @@ -63,11 +63,11 @@ class AddDistrictFieldValue { // } //第三方均价数据填补 -// for(def city : citys){ -//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) -// def rMap = getMatchDistrict(city) -// createResultValue(dists, rMap) -// } + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) + def rMap = getMatchDistrict(city) + createResultValue(dists, rMap) + } // for(def city : citys){ // def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"]) @@ -103,52 +103,52 @@ class AddDistrictFieldValue { // } - def citys = ["上海"] - def updateIndex = 0 - def rets = [] - rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"] - for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"]) - def total = dists.size() - - for(def dist : dists){ - if(dist.household_count){ - def ret = [] - ret << dist._id.toString() - ret << dist.city - ret << dist.region - ret << dist.name - ret << dist.household_count - ret << dist.loudong_count - def details = findDistrictDetail(dist) - if(details){ - for(def d : details){ - def region = RegionData.formatRegion(city, d.region) - if(d.region == region){ - if(dist.household_count && dist.household_count==d.house_count){ - ret << d.loudong_count - ret << "链家" -// updateIndex++ -// def udata = new BasicDBObject() -// udata.put("household_count", d.house_count.toInteger()) -// udata.put("m_date", new Date()) -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - } - } - } - }else{ - ret << "" - ret << "" - } - - rets << ret - } - } - ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据") - } +// def citys = ["上海"] +// def updateIndex = 0 +// def rets = [] +// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"] +// for(def city : citys){ +// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"]) +// def total = dists.size() +// +// for(def dist : dists){ +// if(dist.household_count){ +// def ret = [] +// ret << dist._id.toString() +// ret << dist.city +// ret << dist.region +// ret << dist.name +// ret << dist.household_count +// ret << dist.loudong_count +// def details = findDistrictDetail(dist) +// if(details){ +// for(def d : details){ +// def region = RegionData.formatRegion(city, d.region) +// if(d.region == region){ +// if(dist.household_count && dist.household_count==d.house_count){ +// ret << d.loudong_count +// ret << "链家" +//// updateIndex++ +//// def udata = new BasicDBObject() +//// udata.put("household_count", d.house_count.toInteger()) +//// udata.put("m_date", new Date()) +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// } +// } +// } +// }else{ +// ret << "" +// ret << "" +// } +// +// rets << ret +// } +// } +// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据") +// } } @@ -303,6 +303,14 @@ class AddDistrictFieldValue { def createResultValue(dists, rMap){ def updateIndex = 0 def total = dists.size() + def addressIndex = 0 + def allAreaIndex = 0 + def buildAreaIndex = 0 + def virescence_percent_index = 0 + def cubage_percent_index = 0 + def finish_building_date_index = 0 + def household_count_index = 0 + def car_site_count_index = 0 for(def d : dists){ def key = d.city +"#"+d.region+"#"+d.name def sourceMap = rMap.get(key) @@ -311,12 +319,14 @@ class AddDistrictFieldValue { if(!d.address){ def lastValue = getValuesByField(sourceMap,"address", d.estate_type2) if(lastValue){ + addressIndex++ udata.put("address", lastValue) } } if(!d.all_area){ def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2) if(lastValue){ + allAreaIndex++ udata.put("all_area", lastValue) } } @@ -324,6 +334,7 @@ class AddDistrictFieldValue { if(!d.build_area){ def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2) if(lastValue){ + buildAreaIndex++ udata.put("build_area", lastValue) } } @@ -331,6 +342,7 @@ class AddDistrictFieldValue { if(!d.virescence_percent){ def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2) if(lastValue){ + virescence_percent_index ++ udata.put("virescence_percent", lastValue) } } @@ -338,6 +350,7 @@ class AddDistrictFieldValue { if(!d.cubage_percent){ def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2) if(lastValue){ + cubage_percent_index ++ udata.put("cubage_percent", lastValue) } } @@ -345,6 +358,7 @@ class AddDistrictFieldValue { if(!d.finish_building_date){ def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2) if(lastValue){ + finish_building_date_index ++ udata.put("finish_building_date", lastValue) } } @@ -352,6 +366,7 @@ class AddDistrictFieldValue { if(!d.household_count){ def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2) if(lastValue){ + household_count_index ++ udata.put("household_count", lastValue) } } @@ -359,6 +374,7 @@ class AddDistrictFieldValue { if(!d.car_site_count){ def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2) if(lastValue){ + car_site_count_index ++ udata.put("car_site_count", lastValue) } } @@ -373,6 +389,15 @@ class AddDistrictFieldValue { } } } + + println "addressIndex "+ addressIndex + println "allAreaIndex "+ allAreaIndex + println "buildAreaIndex "+ buildAreaIndex + println "virescence_percent_index "+ virescence_percent_index + println "cubage_percent_index "+ cubage_percent_index + println "finish_building_date_index "+ finish_building_date_index + println "household_count_index "+ household_count_index + println "car_site_count_index "+ car_site_count_index } def getValuesByField(sourceMap, field, estateType){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy index 13b8fdde9..a36cf12a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy @@ -17,7 +17,7 @@ class TempMatchDistrict { } def stat(){ - def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] + def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"] def total = 0 for(def city : citys){ def count = CommonDatautils.findAllDistrictCount("district", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 6b778545b..44f6d4a9b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -38,8 +38,8 @@ class FangjiaClientUtils { } public static def sendDistrictPriceCalculate(def city, def region, def name, def count){ - def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" -// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" +// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" + def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" def content = null; try{ content = new HttpRequest().newGet(url, null, "utf-8") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 4bd387c7a..e6046e703 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -560,7 +560,7 @@ class DistrictIndexBatch extends BaseBatch{ return datas } - def sortByDistance(d, list){ + def sortByDistance(d, list){ for(def nb : list){ def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) nb.distance = distance diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 6d75b18bd..d54ad9fb3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome { static main(args) { def task = new BeiKeCrawlByChrome(); -// task.run() + task.run() // task.run1() task.addDealRegion(); } @@ -110,7 +110,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(!e.text().contains("南京")){ + if(e.text().contains("福州")){ cityMap.put(e.text(), "https:"+href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy index e2c0d9152..4229926a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy @@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.support.ui.ExpectedConditions import org.openqa.selenium.support.ui.WebDriverWait +import javax.swing.JApplet import java.text.DateFormat import java.util.function.Consumer @@ -41,45 +42,137 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca System.setProperty("http.nonProxyHosts", "localhost"); -def doc = getDocument(); -def detailUrls = getDetailLinks(doc); -def rets = batchParse(detailUrls); -saveToDB(rets); + +WebDriver driver = new ChromeDriver() +def proviceMap = getDocument(driver); +def citys = getCity(driver,proviceMap) +def detailUrls = getDetailLinks(driver, citys); -def getDetailLinks(doc){ - def urls = [] - if(doc){ - def eles = JQueryUtils.select(".lots-list .lot-li-a", doc) - if(eles){ - for(def el :eles){ - def href = el.attr("href") - urls << "https://paimai.caa123.org.cn"+href + +def getDetailLinks(driver, cityList){ + + def zxcitys = ["天津市","北京市","上海市","重庆市"] + for(def cityMap : cityList){ + def citys = cityMap.citys + + citys.each{city,id-> + def urls = [] + if(!zxcitys.contains(cityMap.provice)){ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + }else{ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + } + + def doc = Jsoup.parse(driver.getPageSource()); + def count = JQueryUtils.selectText(".count", doc) + def totalPage = getTotalPage(count, 12) + if(totalPage){ + for(def i=1;i<=totalPage;i++){ + doc = Jsoup.parse(driver.getPageSource()); + def eles = JQueryUtils.select(".lots-list .lot-li-a", doc) + if(eles){ + for(def el :eles){ + def href = el.attr("href") + def map = [:] + map.province = cityMap.provice + map.city = city + map.url = "https://paimai.caa123.org.cn"+href + urls << map + } + } + try{ + driver.findElement(By.className("next")).click() + }catch(Exception e){ + Thread.sleep(8000) + driver.findElement(By.className("next")).click() + } + } + println city +" "+urls.size() + def rets = batchParse(urls); + saveToDB(rets); } } + println cityMap.provice +" 完成" + } - return urls; + + //return urls; } -def getDocument(){ - WebDriver driver = new ChromeDriver() +def getTotalPage(def total, def pageSize){ + if(total){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + return 0; +} + +def getCity(driver, proviceMap){ + def rows = [] + def zxcitys = ["天津市","北京市","上海市","重庆市"] + proviceMap.each {k,v-> + if(!zxcitys.contains(k)){ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); + def doc = Jsoup.parse(driver.getPageSource()); + def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc) + def cityMap = [:] + for(def c : cityEles){ + def cityid = c.attr("data-cityid") + def city = c.text() + if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){ + cityMap.put(city, cityid) + } + + } + def map = [:] + map.provice = k + map.provice_id = v + map.citys = cityMap + rows << map + }else{ + def map = [:] + map.provice = k + map.provice_id = v + def cityMap = [:] + cityMap.put(k, v) + map.citys = cityMap + rows << map + } + } + return rows +} + +def getDocument(driver){ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); WebDriverWait wait = new WebDriverWait(driver,20) wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list")))); - def doc = Jsoup.parse(driver.getPageSource()); - - - return doc; + def provinceMap = [:] + if(doc){ + def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc) + for(def ele : proviceEles){ + def provice = ele.text() + def value = ele.attr("data-proviceid") + if(value && provice!="北京市" && provice!="天津市"){ + provinceMap.put(provice, value) + } + } + } + return provinceMap; } def batchParse(def urls){ WebDriver driver = new ChromeDriver() def ret = [] - for(def url :urls){ - driver.get(url); + for(def map :urls){ + driver.get(map.url); def doc = Jsoup.parse(driver.getPageSource()); - def res = parse(url, doc); + def res = parse(map.province, map.city, map.url, doc); if(res){ ret << res } @@ -87,111 +180,126 @@ def batchParse(def urls){ return ret; } -def parse(url,doc){ +def parse(province, city, url,doc){ if(doc){ def result = [:] -// def address = JQueryUtils.selectText(["#pro_bid_name"], doc) -// result.address = address - if(result.address){ - def rMap = OpenApiUtils.matchDistrict(result.address) - if(rMap) { - result.matchCity = rMap.matchCity - result.matchRegion = rMap.matchRegion - result.matchName= rMap.matchName - result.matchAddress = rMap.matchAddress + result.province = province + result.city = city + def address = JQueryUtils.selectText(["#pro_bid_name"], doc) + result.address = address + if(result.address && !result.address.contains("测试")) { + def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num } - } - def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc) - if(finishDateStr) { - result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss"); - } + result.status = JQueryUtils.selectText([".bd_state"], doc) + result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc) + result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc) - def curPriceStr = JQueryUtils.selectText([".colorred"], doc) - if(curPriceStr) { - curPriceStr = curPriceStr.replace(",","") - result.tradePrice = curPriceStr.toInteger() - } - - def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc) - if(evaluationPriceStr) { - evaluationPriceStr = evaluationPriceStr.replace(",", "") - if(evaluationPriceStr.isInteger()) { - result.evaluationPrice = evaluationPriceStr.toInteger() + def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc) + if (finishDateStr) { + result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm"); } - } - def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc) - if(depositPriceStr) { - depositPriceStr = depositPriceStr.replace(",", "") - if(depositPriceStr.isInteger()) { - result.depositPrice = depositPriceStr.toInteger() + def curPriceStr = JQueryUtils.selectText([".colorred"], doc) + if (curPriceStr) { + curPriceStr = curPriceStr.replace(",", "") + result.tradePrice = curPriceStr.toDouble().toInteger() } - } - def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc) - if(addPriceStr) { - addPriceStr = addPriceStr.replace(",", "") - if(addPriceStr.isInteger()) { - result.addPrice = addPriceStr.toInteger() - } - } - - def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc) - if(startPriceStr) { - startPriceStr = startPriceStr.replace(",", "").replace("元", "") - if(startPriceStr.isInteger()) { - result.startPrice = startPriceStr.toInteger() - } - } - - result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc) - result.retain = JQueryUtils.selectText(["#retain"], doc) - - def imgs =JQueryUtils.select(".sf-pic-slide img", doc) - if(imgs) { - def imgLinks = [] - for(def img : imgs) { - def href = img.attr("src") - if(href.contains("http")) { - imgLinks << href - }else { - imgLinks << url.substring(0, url.indexOf(":")+1)+href + def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc) + if (evaluationPriceStr) { + evaluationPriceStr = evaluationPriceStr.replace(",", "") + if (evaluationPriceStr.isNumber()) { + result.evaluationPrice = evaluationPriceStr.toDouble().toInteger() } } - result.imgUrls = imgLinks - } - def desc = JQueryUtils.selectText(["#J_desc"], doc) - if(!desc){ - desc = JQueryUtils.selectText(["#pro_bid_name"], doc) - } - if(desc) { - if(desc.contains("别墅")) { - result.address = desc.substring(0, desc.indexOf("别墅")+2) - }else if(desc.contains("号楼")) { - result.address = desc.substring(0, desc.indexOf("号楼")+2) - }else if(desc.contains("房屋")) { - result.address = desc.substring(0, desc.indexOf("房屋")+2) + def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc) + if (depositPriceStr) { + depositPriceStr = depositPriceStr.replace(",", "") + if (depositPriceStr.isNumber()) { + result.depositPrice = depositPriceStr.toDouble().toInteger() + } } - } - def descr = JQueryUtils.selectText(["body"], doc) - if(descr) { - def area = getAreaNumber(descr, 5000); - if(area){ - result.area = area.toDouble() + def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc) + if (addPriceStr) { + addPriceStr = addPriceStr.replace(",", "") + if (addPriceStr.isNumber()) { + result.addPrice = addPriceStr.toDouble().toInteger() + } } - } - if(result.area && result.tradePrice){ - result.tradeAvgPrice = (int)(result.tradePrice/result.area) - } - result.content = DataUtil.zip(doc.toString()) - result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() - result.url = url - return result; + def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc) + if (startPriceStr) { + startPriceStr = startPriceStr.replace(",", "").replace("元", "") + if (startPriceStr.isNumber()) { + result.startPrice = startPriceStr.toDouble().toInteger() + } + } + + result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc) + result.retain = JQueryUtils.selectText(["#retain"], doc) + result.lookCount = JQueryUtils.selectText(".look_count span", doc) + + def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc) + if(startDateStr){ + startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim() + } + + result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr) + def imgs = JQueryUtils.select(".sf-pic-slide img", doc) + if (imgs) { + def imgLinks = [] + for (def img : imgs) { + def href = img.attr("src") + if (href.contains("http")) { + imgLinks << href + } else { + imgLinks << url.substring(0, url.indexOf(":") + 1) + href + } + } + result.imgUrls = imgLinks + } + + def desc = JQueryUtils.selectText(["#J_desc"], doc) + if (!desc) { + desc = JQueryUtils.selectText(["#pro_bid_name"], doc) + } + if (desc) { + if (desc.contains("别墅")) { + result.address = desc.substring(0, desc.indexOf("别墅") + 2) + } else if (desc.contains("号楼")) { + result.address = desc.substring(0, desc.indexOf("号楼") + 2) + } else if (desc.contains("房屋")) { + result.address = desc.substring(0, desc.indexOf("房屋") + 2) + } + } + + def descr = JQueryUtils.selectText(["body"], doc) + if (descr) { + def area = getAreaNumber(descr, 5000); + if (area) { + result.area = area.toDouble() + } + } + if (result.area && result.tradePrice) { + result.tradeAvgPrice = (int) (result.tradePrice / result.area) + } + + result.content = DataUtil.zip(doc.toString()) + result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() + result.url = url + return result; + } } return null } @@ -209,9 +317,11 @@ def saveToDB(rets){ } if(dbo){ - obj.remove("url") + obj.m_date = new Date() paimaiResultColl.update(q, new BasicDBObject('$set', obj)) }else{ + obj.c_date = new Date() + obj.m_date = new Date() paimaiResultColl.save(obj) } } @@ -229,6 +339,12 @@ def getAreaNumber(input, maxArea) { def area = null; for (def val : list) { + if(val.toString().contains(" ")){ + val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size()) + } + if(val.contains("室")){ + val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size()) + } def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", ""); if (maxArea) { if (v.toDouble() < maxArea) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 76d63a6a5..f1750cfa9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -417,11 +417,12 @@ public class LianjiaCrawlByChrome { def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] if(cityEles){ - for(def e : cityEles){ + for(def e : cityEles) { def href = e.attr("href") -// if(!href.contains("fang.lianjia") && e.text().contains("福州")){ + if (!href.contains("fang.lianjia") && e.text().contains("福州")) { cityMap.put(e.text(), href) } + } } return cityMap } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index 44664323e..e35e9f251 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -7,34 +7,147 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver +import java.text.SimpleDateFormat import java.util.regex.Matcher import java.util.regex.Pattern -import java.util.zip.GZIPInputStream -import java.util.zip.GZIPOutputStream -import org.apache.commons.lang.StringUtils; -import java.text.SimpleDateFormat; class PaimaiTest { + + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + static main(args) { def task = new PaimaiTest(); task.run(); } def run(){ - def url = "http://www.gpai.net/sf/item2.do?Web_Item_ID=23134" - def con = find(url); - parser(con, url); + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def url = "https://s.gpai.net/sf/search.do?at=376" + def provinces = getProvice(url, driver) + def cityUrls = getCity(provinces, driver) + getDetailUrls(cityUrls, driver) + } + def getDetailUrls(cityUrls, driver){ + for(def cMap : cityUrls){ + driver.get("https:"+cMap.url) + def doc = Jsoup.parse(driver.getPageSource()); + def total = JQueryUtils.selectText("span:contains(共有拍品) label", doc) - def parser(s, url){ - Document doc = Jsoup.parse(s); + if(total){ + def totalPage = getTotalPage(total.toInteger(), 20) + def detailDatas = [] + for(def i=1;i<=totalPage;i++){ + driver.get("https:"+cMap.url+"&page="+i) + doc = Jsoup.parse(driver.getPageSource()); + def detailEles = JQueryUtils.select(".block-wrap:contains(共有拍品) .filt-result-list li .item-tit", doc) + for(def ele : detailEles){ + def dMap = [:] + dMap.proince = cMap.proince + dMap.city = cMap.city + dMap.url = ele.select("a").attr("href") + detailDatas << dMap + } + } + + def results = parser(detailDatas, driver, cMap.proince, cMap.city) + saveData(results) + println cMap.proince+" "+cMap.city+" "+results.size() + } + } + } + + + def getTotalPage(def total, def pageSize){ + if(total){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + return 0; + } + + def getCity(datas, driver){ + def citys = ["上海","北京","天津","重庆"] + def cityList = [] + for(def data : datas){ + driver.get("https:"+data.href) + def doc = Jsoup.parse(driver.getPageSource()); + if(citys.contains(data.proince)){ + def cityMap = [:] + cityMap.proince = data.proince + cityMap.city = data.proince + cityMap.url = data.href + cityList << cityMap + }else{ + def links = JQueryUtils.select(".condition .s-subtab a", doc) + for(def ele : links){ + if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){ + def cityMap = [:] + cityMap.proince = data.proince + cityMap.city = ele.text() + cityMap.url = ele.attr("href") + cityList << cityMap + } + + } + } + } + return cityList + } + + def getProvice(url, driver){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def datas = [] + def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"] + if(doc){ + def eles = JQueryUtils.select(".condition a", doc) + for(def e : eles){ + def data = [:] + if(!keys.contains(e.text())){ + data.proince = e.text() + data.href = e.attr("href") + datas << data + } + } + } + return datas + } + + def parser(detailDatas, driver, proince, city){ + def results = [] + for(def m : detailDatas){ + try{ + driver.get("https:"+m.url) + Thread.sleep(3000) + def doc = Jsoup.parse(driver.getPageSource()); + def obj = parsering(doc, m.url) + if(obj){ + obj.proince = proince + obj.city = city + results << obj + } + }catch(Exception e){ + println m.url+" 解析有问题!" + } + + } + return results + } + + def parsering(doc, url){ def result = [:] if(doc) { - def title = JQueryUtils.selectText([".d-m-title"], doc) if(!title) { title = JQueryUtils.selectText([".DivItemName"], doc) @@ -48,6 +161,9 @@ class PaimaiTest { } result.title = title result.status = status + if(title.contains("北京市朝阳区东三环中路7号4号楼28层3201")){ + println title + } if(title) { def newStr =title @@ -76,7 +192,7 @@ class PaimaiTest { if(curPriceStr.contains("成交价:")) { def tradePriceStr = JQueryUtils.selectText([".d-m-price .price-red"], doc).replace("元","") if(tradePriceStr) { - result.tradePrice = tradePriceStr.toInteger() + result.tradePrice = tradePriceStr.toDouble().toInteger() } } } @@ -85,23 +201,28 @@ class PaimaiTest { curPriceStr = JQueryUtils.selectText(["#ItemArea .red"], doc) if(curPriceStr) { curPriceStr = curPriceStr.replace(",", "").replace("元","").replace("--","") - result.tradePrice = curPriceStr.toInteger() + result.tradePrice = curPriceStr.toDouble().toInteger() } } - def enrollCount = JQueryUtils.selectText([".peoples-infos span:eq(0) b"], doc) - if(enrollCount) { + def enrollCount = JQueryUtils.selectText([".peoples-infos span:contains(人报名) b"], doc) + if(enrollCount && enrollCount.isNumber()) { result.enrollCount = enrollCount.toInteger() } - def callCount = JQueryUtils.selectText([".peoples-infos span:eq(1) b"], doc) - if(callCount) { + def callCount = JQueryUtils.selectText([".peoples-infos span:contains(人设置提醒) b"], doc) + if(callCount && callCount.isNumber()) { result.callCount = callCount.toInteger() } - def lookCount = JQueryUtils.selectText([".peoples-infos span:eq(2) b"], doc) - if(lookCount) { - result.lookCount = lookCount.toInteger() + def lookCount = JQueryUtils.selectText([".peoples-infos span:contains(人围观) b"], doc) + if(lookCount && lookCount.isNumber()) { + try{ + result.lookCount = lookCount.toInteger() + }catch(Exception e){ + println lookCount + } + } def finishDateStr = JQueryUtils.selectText(["p:contains(拍卖竞价确认时间:)"], doc) @@ -140,7 +261,7 @@ class PaimaiTest { }else if(str.contains("保证金")) { def v = str.replace("保证金:", "").replace("元", "").replace(",", "").replace("--", "") if(v) { - result.depositPrice = v.toInteger() + result.depositPrice = v.toDouble().toInteger() } }else if(str.contains("延时周期")) { @@ -184,6 +305,47 @@ class PaimaiTest { } } } + + if(detailMap && !detailMap.get("拍品名称")){ + detailMap = [:] + for(def tr : trs) { + + if(tr.text().contains("拍品名称")){ + def td1 = tr.select("td:contains(拍品名称)+td").text() + detailMap.put("拍品名称", td1) + } + + if(tr.text().contains("权利来源")){ + def td1 = tr.select("td:contains(权利来源)+td").text() + detailMap.put("权利来源", td1) + } + + if(tr.text().contains("权证情况")){ + def td1 = tr.select("td:contains(权证情况)+td").text() + detailMap.put("权证情况", td1) + } + + if(tr.text().contains("拍品所有人")){ + def td1 = tr.select("td:contains(拍品所有人)+td").text() + detailMap.put("拍品所有人", td1) + } + + if(tr.text().contains("房屋情况")){ + def td1 = tr.select("td:contains(房屋情况)+td").text() + detailMap.put("房屋情况", td1) + } + + if(tr.text().contains("权利限制情况及瑕疵情况")){ + def td1 = tr.select("td:contains(权利限制情况及瑕疵情况)+td").text() + detailMap.put("权利限制情况及瑕疵情况", td1) + } + + if(tr.text().contains("拍品介绍")){ + def td1 = tr.select("td:contains(拍品介绍)").text() + detailMap.put("拍品介绍", td1) + } + } + } result.detailMap = detailMap if(!result.address) { result.address = detailMap.get("拍品名称") @@ -248,17 +410,18 @@ class PaimaiTest { } } - result.content = DataUtil.zip(s) +// result.content = DataUtil.zip(s) def el = JQueryUtils.select([".d-block table:eq(1)"], doc) if(!el) { el = JQueryUtils.select([".d-title:contains(标的物介绍)+div"], doc) } - result.content = DataUtil.zip(s) +// result.content = DataUtil.zip(s) result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString() result.url = url result.mDate = new Date(); + return result; } def find(url){ @@ -374,4 +537,32 @@ class PaimaiTest { } return null; } + + def saveData(def results){ + + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy index 40b17049e..084a3d183 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy @@ -5,10 +5,15 @@ import com.fangjia.core.util.DataUtil import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +class BlankEvaluationBatch { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -class EvaluationBatch { static main(args) { - def task = new EvaluationBatch(); + def task = new BlankEvaluationBatch(); task.run() } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy new file mode 100644 index 000000000..e5ffbd54f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy @@ -0,0 +1,205 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class CqfdcChromeCrawl { + + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) + + + static main(args) { + def task = new CqfdcChromeCrawl(); + task.run() + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def list = getListDatas(driver) + getDetails(driver, list); + + } + + + + def getDetails(driver, list){ + def index = 0 + for(def map_old : list){ + def datas = [] + index++ + def url = "http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=${map_old.building_id}&block=${map_old.house_num_desc}" +// driver.get(url) +// def doc = driver.getPageSource() + def cityDoc = getDocment(url, driver) + def trs = JQueryUtils.select(".building_table tr", cityDoc) + def unit1 = null + def unit2 = null + def unit3 = null + def unit1_text = null + def unit2_text = null + def unit3_text = null + for(def i=0;i + map.put(k,v) + } + datas << map + } + } + } + } + saveDatas(datas) + println index+" "+map_old.name+" "+map_old.house_num_desc +" "+datas.size() + } + } + + def getDocment(def url, def driver){ + def stop = false + def doc = null + def error = 0 + while (!stop || error>=10){ + try{ + driver.get(url) + def docment = driver.getPageSource() + if(docment){ + doc = Jsoup.parse(docment) + stop = true; + }else{ + error ++ + } + }catch(Exception e){ + error ++ + } + } + return doc; + } + + def getListDatas(driver){ + driver.get("http://cqfdc.cq315house.com/HtmlPage/PresaleDetail.html") + def datas = [] + for(def i=2;i<=208;i++){ + def docment = Jsoup.parse(driver.getPageSource()); + def trs = JQueryUtils.select("#searchbody tr", docment) + for(def tr : trs){ + def tds = tr.select("td") + + def hrefs = tds.get(3).select("a") + for(def href : hrefs){ + def map = [:] + map.house_num_desc = href.text() + def str = href.attr("href") + def building_id= str.substring(str.lastIndexOf("['")+2, str.lastIndexOf("','")) + map.building_id = building_id + + map.city = "重庆" + map.name = tds.get(0).text() + map.company = tds.get(1).text() + map.license_key = tds.get(2).text() + datas << map + } + } + println i +" "+datas.size() + try{ + driver.findElement(By.id("nextpage")).click() + }catch(Exception e){ + Thread.sleep(8000) + driver.findElement(By.id("nextpage")).click() + } + + } + println "列表:"+datas.size() + return datas; + } + + + def saveDatas(def datas){ + def rows = [] + def keys = [] + for(def d : datas){ + def q = new BasicDBObject() + q.put("city", d.city) + q.put("name",d.name) + q.put("house_num_desc", d.house_num_desc) + q.put("room_num", d.room_num) + q.put("unit_num",d.unit_num) + def obj = loudongConn.findOne(q) + if(!obj){ + def key = d.house_num_desc +"#"+d.unit_num+"#"+d.room_num + if(!keys.contains(key)){ + def udata = new BasicDBObject() + d.each {k,v-> + udata.put(k,v) + } + udata.put("c_date", new Date()) + rows << udata + keys << key + } + } + } + if(rows){ + loudongConn.insert(rows); + println "save: "+rows.size() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy index 31ab75ba6..8684414c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy @@ -30,8 +30,8 @@ class HouseBookChromeCrawl { WebDriver driver = new ChromeDriver() def listPageUrl = getIndexPage(driver) def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl) - keywords.remove("城南家园") - keywords.remove("康居西城") +// keywords.remove("城南家园") +// keywords.remove("康居西城") println city +"共 "+keywords.size()+" 个小区" getDistrictIndex(driver, keywords) @@ -61,15 +61,15 @@ class HouseBookChromeCrawl { def getAllDist(driver, url, page){ def keywords = [] - for(def i=1;i<=page;i++){ + for(def i=73;i<=page;i++){ driver.get(url+"?selectPages="+i) def doc = Jsoup.parse(driver.getPageSource()) def eles = JQueryUtils.select(".title-area .title-text", doc) for(def e : eles){ def text = e.text() - if(text=="富州北郡"){ - println i - } +// if(text=="富州北郡"){ +// println i +// } keywords << text } } @@ -206,7 +206,7 @@ class HouseBookChromeCrawl { detail.estate_type = house_type } if(detail){ - =-09ql.floor = floor + detail.floor = floor rows << detail } } @@ -214,7 +214,7 @@ class HouseBookChromeCrawl { } } return rows; - }+*-* + } def getIndexPage(driver){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy new file mode 100644 index 000000000..8a9e81f31 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.thirdPriceCrawl.UserAgents +import com.fangjia.script.util.ABuYunUtil +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.jsoup.Jsoup +import org.jsoup.select.Elements + +class HouseBookCrawl { + + public def CHARSET = "utf-8"; + + static main(args) { + def task = new HouseBookCrawl() + task.run() + } + +// def run(){ +// goTocitypage() +// } +// def goTocitypage(){ +// def url = "http://www.house-book.com.cn/mainAction/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html" +// def headers = [:] +// headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); +// headers.put("User-Agent", UserAgents.get()) +// headers.put("cookCity", "%2525E9%252587%25258D%2525E5%2525BA%252586%2525E5%2525B8%252582") +// headers.put("HBJSESSIONID","afb32a63-796e-4bbc-bf39-0f8cb188079e") +// def params = [:] +// def responseMap = ABuYunUtil.doPostResponse(url, null, headers, CHARSET, false) +// def cookie = getCookie(responseMap) +// } +// +// def getCookie(responseMap) { +// def cookieAuth = null +// try { +// def response = responseMap.get("response"); +// if (StringUtils.isNotEmpty(response)) { +// JSONObject reJson = JSONObject.fromObject(response) +// //登录成功获取COOKIE +// if (reJson.success) { +// responseMap.get("headerFields").each { k, v -> +// if (k && k.equals("Set-Cookie")) { +// if (v) { +// cookieAuth = v.join(';') +// } +// } +// } +// } +// } +// } catch (Exception e) { +// println "goApiLogin(${responseMap})发生异常, 异常信息为:" + e.getMessage() +// } +// println("最终构造cookieAuth为:" + cookieAuth) +// return cookieAuth +// } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy new file mode 100644 index 000000000..1e7978919 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy @@ -0,0 +1,147 @@ +package com.fangjia.script.loudong + +import net.sourceforge.tess4j.ITesseract +import net.sourceforge.tess4j.Tesseract +import net.sourceforge.tess4j.util.ImageHelper +import org.apache.commons.io.FileUtils +import org.ehcache.xml.model.TimeUnit +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.OutputType +import org.openqa.selenium.Point +import org.openqa.selenium.TakesScreenshot +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver + +import javax.imageio.ImageIO +import java.awt.image.BufferedImage + +public class ImageTest { + + public static void main(String[] args) throws IOException { +// File imageFile = new File("D:\\gaojuan\\img\\WaterMark.gif"); +// Tesseract instance = new Tesseract(); +// + +// //将验证码图片的内容识别为字符串 +// String result = instance.doOCR(imageFile); +// println result + + //1.获取网页的浏览器driver + //(1)定义url + String url="http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=2l/xVhsMJPAZKivgt8ciOA&block=3%E5%8F%B7%E6%A5%BC#"; + //(2)设置driver驱动chromedriver.exe(版本:2.39)路径(相对路径) + String path = System.getProperty("user.dir"); + System.setProperty("webdriver.chrome.driver","C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + //(3)建立selenium 驱动 + //WebDriver driver = new ChromeDriver(options); + WebDriver driver = new ChromeDriver(); + //(4)设置窗口最大化及网页等待时间 + driver.manage().window().maximize(); //窗口最大化 +// driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS); + //(5)启动driver浏览网页 + driver.get(url); //输入网址 + Thread.sleep(1000); + + driver.findElement(By.id("cell1-11-2")).click() + Thread.sleep(1000); + + def docment = Jsoup.parse(driver.getPageSource()); + println docment + + WebElement ele = driver.findElement(By.id("vimg")); + + File screenshot = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE); + BufferedImage pic1 = ImageIO.read(screenshot); + Point point = ele.getLocation(); + + //(2)输入验证码 + File srcfile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); + FileUtils.copyFile(srcfile, new File("picture/source.png")); + //原图片 + //String soursePicPath = "D:\\data_dev\\test\\test.png"; + String soursePicPath = "picture/source.png"; + File sourcePic = new File(soursePicPath); + int eleWid = ele.getSize().getWidth() + int eleHeiht = ele.getSize().getHeight() + + BufferedImage pic2 = pic1.getSubimage(point.getX(), point.getY(), eleWid, eleHeiht); + ImageIO.write(pic2, "jpg", sourcePic); + + //File desImage = new File("D:\\data_dev\\test\\test02.png"); + File desImage = new File("picture/answer.png"); + + + + //加载待读取图片 + //File imageFile = new File("D:\\data_dev\\test\\test02.png"); + File imageFile = new File("picture/answer.png"); + BufferedImage image = ImageIO.read(imageFile); + //对图片进行处理 + image = convertImage(image); + //创建tess对象 + ITesseract instance = new Tesseract(); + //设置训练文件目录 + //instance.setDatapath("E:\\workspace\\workspace_idea\\com_mth_data\\m-kuaidi100-data\\tessdata"); + String path02 = System.getProperty("user.dir"); + instance.setDatapath(path02 + "\\tessdata"); + //设置训练语言 + //instance.setLanguage("chi_sim"); + instance.setLanguage("eng"); + //执行转换 + String s = instance.doOCR(image); + + System.out.println(s); + String[] split = s.replace(" ", "").split("=|:"); + String a = split[split.length - 1].trim(); + char[] chars = a.toCharArray(); + if (chars.length == 2) { + if (chars[0] == '7') { + chars[0] = '1'; + } + } + a = String.valueOf(chars); + + System.out.println(a); + String b = s.substring(0, 1); + System.out.println(b); + String answer = String.valueOf(Integer.parseInt(a.trim()) - Integer.parseInt(b.trim())); + + + WebElement input2 = driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_TextBox2\"]")); + input2.sendKeys(answer); + Thread.sleep(4000); + //(3)点击"查询"按钮 + driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_Button1\"]")).click(); + + String SecondtHandle = driver.getWindowHandle(); //首先得到最先的窗口 权柄 + for (String winHandle1 : driver.getWindowHandles()) { //得到浏览器所有窗口的权柄为Set集合,遍历 + if (!winHandle1.equals(SecondtHandle)) { //如果为 最先的窗口 权柄跳出 + driver.close(); + driver.switchTo().window(winHandle1); //如果不为 最先的窗口 权柄,将 新窗口的操作权柄 给 driver + System.out.println(driver.getCurrentUrl()); //打印是否为新窗口 + } + } + + Thread.sleep(4000); + + //4.关闭浏览器 + driver.close(); + + } + + + //对图片进行处理 - 提高识别度 + public static BufferedImage convertImage(BufferedImage image) throws Exception { + //按指定宽高创建一个图像副本 + //image = ImageHelper.getSubImage(image, 0, 0, image.getWidth(), image.getHeight()); + //图像转换成灰度的简单方法 - 黑白处理 + image = ImageHelper.convertImageToGrayscale(image); + //图像缩放 - 放大n倍图像 + image = ImageHelper.getScaledInstance(image, image.getWidth() * 15, image.getHeight() * 15); + return image; + } + + +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index d7d81e90e..9362389ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -191,24 +191,24 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"] - for(def city : citys){ - def str = cityList.get(city) - for(def i=0;i<1;i++){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - } - } - } +// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"] +// for(def city : citys){ +// def str = cityList.get(city) +// for(def i=0;i<1;i++){ +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); +// } +// } +// } - /*cityList.each{city, str-> + cityList.each{city, str-> for(def i=0;i<1;i++){ - if(citys.contains(city)){ +// if(citys.contains(city)){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); if(districts){ def arrs = str.split("\\|") @@ -218,9 +218,9 @@ class Esf58Crawl extends BaseBatch{ addCrawlKeys(city); addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); } - } +// } } - }*/ + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 1ce2d438f..0ca7c1afb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,10 +15,10 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); -// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", -// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] + def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", + "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] //"fang.com":"搜房", - def sourceMap = ["fang.com":"搜房"] +// def sourceMap = ["fang.com":"搜房"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index db48455df..bd0fbb3f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202201); - def count2 = findTotalCount(source, 202202); + def count1 = findTotalCount(source, 202202); + def count2 = findTotalCount(source, 202203); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index bfa7f8ec7..f6399a2dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.util +import org.apache.commons.lang.StringUtils + import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; @@ -995,4 +997,42 @@ class DateUtils { return rtn; } + public static Date parseString2Date(String str){ + if(StringUtils.isBlank(str)){ + return null + } + str = str.replaceAll("^\\s+", "").replaceAll("\\s+\$", "") + + Date now = new Date() + if(str.contains("小时")||str.contains("分钟") || str.contains("秒") || str.contains("当前")|| str.contains("今天")){ + return org.apache.commons.lang3.time.DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH) + }else if(str.contains("昨天")) { + return add(now, -1) + }else if(str.contains("天")){ + int days = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*days) + }else if(str.contains("月前")){ + int month = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*month*30) + }else if(str.contains("年前")){ + int years = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*years*365) + }else{ + if(str.size()<10){ + if(str.contains("-")){ + str = (now.getYear()+1900)+"-"+str + } + }else{ + if(str.contains("月") && str.contains("日") && !str.contains("年")){ + str = (now.getYear()+1900)+"年"+str.substring(0, str.indexOf("日")+1) + } + } + if(str.contains("年")){ + return DateUtils.parseFromTightByPattern(str, "yyyy年MM月dd日") + }else{ + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + } + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index e0d8713b6..3810aa9f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220201", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220301", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220301", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220401", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0