diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 407d37700..6e1b76aa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220718, 20220819); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index e8f85a1b7..0715fc907 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -26,7 +26,8 @@ class FjwPricePkCcbPrice { static main(args) { def task = new FjwPricePkCcbPrice() // task.run() - task.updatePrice() +// task.updatePrice() + task.addFjwPrice() } def updatePrice(){ @@ -41,6 +42,21 @@ class FjwPricePkCcbPrice { }) } + def addFjwPrice(){ + ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d-> + if(!d['M']){ + def dist = CommonDatautils.findDistrictById(d['D']) + if(dist){ + def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd")) + if(dbo){ + d['M'] = dbo.avg_price + } + } + } + + }) + } + def run(){ def ccbPrices = getCommunityBasePriceGj(202208) println "ccb总量:"+ccbPrices.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 261a7f702..c4a27815e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202207//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202205 - def endDate = 202207 + def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202206 + def endDate = 202208 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 61523694a..33b350c5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3166,8 +3166,7 @@ class CommonDatautils { DBObject dbo; def prices = [] def index = 0 - while(cur.hasNext()) { - def obj = cur.next() + for(def obj : cur) { if(index == 0){ dbo = obj } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 08aceca5b..a624061d9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -149,7 +149,7 @@ class FzDataResultStat { def exportRegion(city, months){ - def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","闽侯","福清","连江","闽清","罗源","永泰","平潭"]; + def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; def datas =[] def header = ["城市", "区域"] for(def m : months){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index c9a2c19be..7cdcc589a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202207.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy similarity index 97% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy index d81601ebb..960b75f38 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CityStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/CityStat.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.gj.test +package com.fangjia.script.gj.commStat import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils @@ -37,11 +37,12 @@ class CityStat { } def dealPriceStat(){ - def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"] + //def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"] + def citys = ["石家庄","贵阳"] def rets = [] - rets << ["城市","2021-1月总量"] - def startDate = DateUtils.parseFromTightByPattern('2021-12-01','yyyy-MM-dd') - def endDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd') + rets << ["城市","2022-4月总量"] + def startDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd') + def endDate = DateUtils.parseFromTightByPattern('2022-02-01','yyyy-MM-dd') for(def c : citys){ def ret = [] ret << c diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy similarity index 92% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy index 4610f71ee..d2d11ec50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DataStatTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/commStat/DataStatTest.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.gj.test +package com.fangjia.script.gj.commStat import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils @@ -29,24 +29,24 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43") -def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"] +//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"] //for(def city : citys){ // //} -ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d-> -// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL) -// if(keys){ -// d['G'] = keys.size() +//ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d-> +//// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL) +//// if(keys){ +//// d['G'] = keys.size() +//// } +// +//// def rets = findHousesByCity(d['A'], "sell", 20210701) +// if(d['A']){ +// def count = findHouseCount(d['A']) +// d['H'] = count // } - -// def rets = findHousesByCity(d['A'], "sell", 20210701) - if(d['A']){ - def count = findHouseCount(d['A']) - d['H'] = count - } - -}) +// +//}) def findHouseCount(city){ @@ -306,3 +306,23 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){ // rows << [c, districts.size(), alias_count, addr_alias_count] //} //ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果") + +//区域级统计 +def city = "合肥" +def regions = CommonDatautils.getCityRegions(city) +def rows = [] +for(def region : regions){ + def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"]) + def alias_count = 0 + def addr_alias_count = 0 + for(def r : districts){ + if(r.alias && r.alias.size()>0){ + alias_count ++ + } + if(r.addr_alias && r.addr_alias.size()>0){ + addr_alias_count++ + } + } + rows << [region, districts.size(), alias_count, addr_alias_count] +} +ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy new file mode 100644 index 000000000..2e9a536ef --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/dongguanbank/DongGuanBankTest.groovy @@ -0,0 +1,135 @@ +package com.fangjia.script.gj.dongguanbank + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import org.apache.lucene.queryparser.classic.QueryParser + +class DongGuanBankTest { + static main(args) { + def task = new DongGuanBankTest() + def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx" +// task.run(path) + task.addPrice(path) + } + + def run(path){ + ExcelUtils.multiThreadingModify(path,{d-> + def city = d['A'].replace("市","") + if(city){ + def region = RegionData.formatRegion(city, d['B']) + def results = OpenApiClient.addressMatch(city, d['F'], "property", false, region, d['C']) + if(results && results.precise>=0){ + d['H'] = results.city + d['I'] = results.district + d['J'] = results.name + d['Q'] = results.propertytype + } + } + + }) + } + + + def addPrice(path){ + ExcelUtils.multiThreadingModify("D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx", { d -> + if(d['H']){ + def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district") + def priceObj = CommonDatautils.queryDistrictPrice(d['H'], d['I'], d['J'], DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd")) + if(priceObj){ + d['M'] = priceObj.avg_price + d['N'] = (priceObj.avg_price * d['G']).toInteger() + d['O'] = ((int)(priceObj.avg_price/100))*100 + if(dist){ + d['R'] = dist.estate_type2 + } + if(priceObj.value_type){ + if(priceObj.value_type == 1){ + d['S'] = "指导价" + }else if(priceObj.value_type == 2){ + d['S'] = "自身样本" + }else if(priceObj.value_type == 3){ + d['S'] = "第三方" + }else if(priceObj.value_type == 4){ + d['S'] = "交易数据" + }else if(priceObj.value_type == 5){ + d['S'] = "周边楼盘" + }else if(priceObj.value_type == 6){ + d['S'] = "推导" + }else if(priceObj.value_type == 9){ + d['S'] = "神经网络" + }else if(priceObj.value_type == 10){ + d['S'] = "无方式" + } + } + } + + if(!d['P']){ + def alias = [] + if(!dist.alias){ + alias = [] + }else{ + alias = dist.alias + } + alias << dist.name + def map = findMaxAvgPrice(dist.city, dist.region, alias) + if(map){ + d['P'] = map.max + d['Q'] = map.min + } + } + + + } + }) + } + + + def findMaxAvgPrice(def city, def region, def alias){ + def set = null + try{ + def aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')" + + set = new HashSet() + ElasticUtils.executeSqlQuery(sql,{data-> + set << data.avg_price + }) + }catch(Exception e){ + e.printStackTrace() + } + + if(set && set.size()>=3){ + def map = [:] + set = set.sort() + map.min = set[0] + map.max = set[set.size()-1] + return map + } + return null + } + + + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def findMinPrice(city){ + def avg_price = null + ElasticUtils.executeSqlQuery("select min(avg_price) as avg_price from seawater where p_date >= 20210901 and city='${city}'", { + avg_price = it.avg_price + }) + return avg_price + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 782185694..42489aafe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -104,9 +104,23 @@ public class BeiKeCrawlByChrome { def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) for(def m : citys){ - //def detailUrls = getPageUrls(driver, cityUrl) - def detailUrls = getPageDeail(driver, m.city, m.url) - println m.city +" 详情链接:"+detailUrls.size() + driver.get(m.url+"/chengjiao/") + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) + if(regions){ + for(def r : regions){ + def regionText = r.text() + def href = m.url+ r.attr("href") + + println "开始抓取【${m.city}】【${regionText}】【${href}】" + def detailUrls = getPageDeail(driver, m.city, href, regionText) + println m.url +" 详情链接:"+detailUrls.size() + } + } + +// def detailUrls = getPageDeail(driver, m.city, m.url) +// println m.city +" 详情链接:"+detailUrls.size() // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) } } @@ -142,12 +156,12 @@ public class BeiKeCrawlByChrome { } - def getPageDeail(driver, city, cityUrl){ + def getPageDeail(driver, city, cityUrl, region){ def stop = false def page = 1 def detailUrls = [] while(!stop){ - def url = cityUrl + "/chengjiao/pg${page}/" + def url = cityUrl + "pg${page}/" try{ driver.get(url) }catch(Exception e){ @@ -164,7 +178,7 @@ public class BeiKeCrawlByChrome { def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl - parse(city, e) + parse(city, e, region) }else{ stop = true; break; @@ -182,7 +196,7 @@ public class BeiKeCrawlByChrome { return detailUrls } - def parse(city,e){ + def parse(city,e, region){ try { def title = e.select(".title a").text() if (title) { @@ -190,7 +204,7 @@ public class BeiKeCrawlByChrome { def arrs = title.split(" ") detail.city = city detail.district_name = arrs[0] - detail.region = findRegionByThirdPary(detail.city, detail.district_name) + detail.region = region//findRegionByThirdPary(detail.city, detail.district_name) if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } @@ -376,13 +390,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy index cef94754f..447a05bb2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ExportPaimai.groovy @@ -177,7 +177,7 @@ class ExportPaimai { def run(){ - def city = "南京" + def city = "合肥" def region = null def datas = findPaimaiData(city,region) println "总量:"+datas.size() @@ -241,7 +241,7 @@ class ExportPaimai { println d.title +" "+d.url } } - ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\南京法拍.xlsx",rows,"数据") + ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据") // batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK") println "生成完成........" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 9bd49bc3b..f975f90b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,9 +34,9 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); -// task.run() + task.run() // task.run1() - task.addDealRegion(); +// task.addDealRegion(); task.standardRegion() // task.updadteDealPriceField() } @@ -152,7 +152,7 @@ public class LianjiaCrawlByChrome { def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") lastMonth = lastMonth +"-01" def q = new BasicDBObject() - q.put("city", "福州") +// q.put("city", "福州") q.put("region", null) q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd"))) // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) @@ -250,7 +250,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-01", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) @@ -469,7 +469,7 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["福州"] + def citys= ["石家庄","贵阳"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 4c4e698f7..2b8b3cf9b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -66,8 +66,8 @@ class PaimaiDataSend { def rmfysszcRun(){ def index = 0 def q = new BasicDBObject() -// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"])) - q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml") + q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"])) +// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml") def dbcursor = rmfysszc_coll.find(q).batchSize(3000) for(def d : dbcursor){ index++ @@ -149,9 +149,9 @@ class PaimaiDataSend { def jdRun(){ def index = 0 def q = new BasicDBObject() - q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"])) + q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"])) // q.put("parser_status", new BasicDBObject('$ne',3)) - q.put("url","https://paimai.jd.com/115669796") +// q.put("url","https://paimai.jd.com/115669796") def dbcursor = jd_coll.find(q).batchSize(3000) for(def d : dbcursor){ index++ @@ -187,19 +187,20 @@ class PaimaiDataSend { def aliZcRun(){ def pm_q = new BasicDBObject() + pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"])) // pm_q.put("city","苏州") // pm_q.put("region","姑苏") - pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm") -// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01"))) -// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE); -// pm_q.put("url", new BasicDBObject('$regex', pattern)) +// pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm") + pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15"))) + Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE); + pm_q.put("url", new BasicDBObject('$regex', pattern)) // pm_q.put("mappingEstateType", null) // pm_q.put("area", new BasicDBObject('$lt', 30)) def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000) println dbcursor.count() for(def d : dbcursor){ -// def flag = sendAliZc(d.url) - def flag = sendAli(d.url) + def flag = sendAliZc(d.url) +// def flag = sendAli(d.url) if(!flag){ println "不存在:"+d.url } @@ -240,7 +241,6 @@ class PaimaiDataSend { byte[] packedMsg = MessageUtils.pack(json) CommonUtils.sendMessage("result2parse2dev", packedMsg) - def udata = new BasicDBObject() udata.put("parser_status", 1) udata.put("m_date", new Date()) @@ -294,7 +294,7 @@ class PaimaiDataSend { def aliFpRun(){ def index = 0 def q = new BasicDBObject() - q.put("city",new BasicDBObject('$in', ["南京","宿迁"])) + q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) // q.put("parser_status", new BasicDBObject('$ne',2)) // q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm") //https://sf-item.taobao.com/sf_item/657131607486.htm @@ -373,10 +373,11 @@ class PaimaiDataSend { def aliZcRun2(){ def index = 0 def q = new BasicDBObject() - q.put("city",new BasicDBObject('$in', ["上海"])) +// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"])) + q.put("city","上海") // q.put("parser_status", new BasicDBObject('$ne',1)) // q.put("region","姑苏") -// q.put("url","https://sf-item.taobao.com/sf_item/586415617368.htm") + q.put("url","https://susong-item.taobao.com/auction/652481594083.htm") def dbcursor = ali_zc_coll.find(q).batchSize(5000) def rets = [] for(def d : dbcursor){ @@ -541,6 +542,11 @@ class PaimaiDataSend { o.mappingAgencyName = d.mappingAgencyName//映射资产类型 o.mappingStatus = d.mappingStatus//映射拍卖状态 o.mappingStage = d.mappingStage + + if(d.startRateDiscount){ + o.startRateDiscount = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, d.startRateDiscount);//起拍折价率 起拍折价率 = 起拍价 / 市场价或评估价 + } + o.diffPrice = d.diffPrice;//差价 差价 = 成交价-市场价或评估价 rets << o } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 48960b876..e1976f1cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -16,19 +16,152 @@ import groovy.sql.Sql import org.bson.types.ObjectId import java.text.SimpleDateFormat +import java.util.regex.Pattern DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018) -def q = new BasicDBObject() -q.put("root", "anjuke.com") -def dbcursor = coll.find(q) -def map = [:] -for(def d : dbcursor){ - map.put(d.city, d.city_short) +def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL) + +//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d-> +// def city = RegionData.formatCity(d['E']) +// if(city && !d['J']){ +// def region = RegionData.formatRegion(city, d['G']) +// d['I'] = city +// d['J'] = region +// } +//}) + + +//ExcelUtils.modify("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx",{d-> +// def city = RegionData.formatCity(d['D']) +// if(!city){ +// city = RegionData.formatCity(d['F']) +// } +// if(city && !d['H']){ +// def region = RegionData.formatRegion(city, d['B']) +// d['G'] = city +// d['H'] = region +// } +//}) + +//def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] +//def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx") + +//def fjwCitys = [] +//for(def d : datas){ +// fjwCitys << d['G'] +//} +//def rows = [] +//for(def d : datas){ +// if(citys.contains(d['G'])){ +// rows << d +// } +//} +//ExcelUtils.write("D:\\数据\\建行\\二期\\统计局行政区划与房价网区域比对V2.xlsx", rows, "区域") + +//for(def c : citys){ +// if(!fjwCitys.contains(c)){ +// println c +// } +//} + +// +//ExcelUtils.write("D:\\数据\\建行\\二期\\(特殊城市)统计局行政区划与房价网区域比对.xlsx", rows, "区域") +// + +//def map = [:] +//def datas1 = ExcelUtils.read("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx") +//for(def row : datas1){ +// if(row['J']){ +// def codeMap = [:] +// codeMap.prov = row['B'] +// codeMap.city = row['F'] +// codeMap.region = row['H'] +// map.put(row['I']+"#"+row['J'], codeMap) +// } +//} +//def datas2 = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx") +//for(def row : datas2){ +// if(row['H']){ +// def codeMap = [:] +// codeMap.prov = row['E'] +// codeMap.city = row['C'] +// codeMap.region = row['A'] +// map.put(row['G']+"#"+row['H'], codeMap) +// } +//} + +//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> +// def key = d['B'] +"#"+d['C'] +// def codeMap = map.get(key) +// if(codeMap && !d['D']){ +// d['D'] = codeMap.prov +// d['E'] = codeMap.city +// d['F'] = codeMap.region +// } +//}) + +ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> + if(!d['D']){ + def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) + if(infos && infos.size()==1){ + def info = infos[0] + d['D'] = info.province_code + d['E'] = info.area_code + d['F'] = info.street_code + } + } +}) + + +def findTownshipInfo(city, region, coll){ + def q = new BasicDBObject() + q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) + q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE)) + + def dbcursor = coll.find(q) + def rows = [] + for(def d : dbcursor){ +// if(d.street_name.contains(region)){ + rows << d +// } + } + return rows } -println map.size() -println map + + + +//def cityMap = CommonDatautils.findAllCityProvince() +//def rows = [] +//rows << ["省", "市", "区域"] +//cityMap.each{c,p-> +// def regions = CommonDatautils.getCityRegions(c) +// for(def r : regions){ +// def row = [] +// row << p +// row << c +// row << r +// rows << row +// } +//} +//ExcelUtils.write("D:\\数据\\亲邻\\房价网省市区.xlsx", rows, "区域") + + + + +//def q = new BasicDBObject() +//q.put("root", "anjuke.com") +//def dbcursor = coll.find(q) +//def map = [:] +//for(def d : dbcursor){ +// map.put(d.city, d.city_short) +//} +//println map.size() +//println map + + //def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164") //DBUtils.seaweed.save(dist) //DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy index 0c8e1fae5..b92188bdd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/TestSendSchoolData.groovy @@ -19,47 +19,47 @@ DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "sch DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018) -ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) - +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) +//Thread.sleep(60*60*1000) //while (true){ -// def q = new BasicDBObject() + def q = new BasicDBObject() // q.put("province",new BasicDBObject('$in', ["江苏","江苏省"])) -// q.put("status",new BasicDBObject('$ne', 1)) -// def dbcursor = school_coll.find(q).limit(1000) -// def rets = [] -// for(def d : dbcursor){ -// JSON json = new JSONObject() -// json.put("project", d.project) -// json.put("url", d.url) -// -// JSON result = new JSONObject() -// result.put("content", d.content) -// def jsonStr = d.map_info -// -// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1) -// if(jsonStr.contains("\n")){ -// jsonStr = jsonStr.replaceAll('\\n','') -// } -// result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr)) -// -// -// result.put("type", d.type) -// json.put("result", result) -// -// -// rets << json -//// if(rets.size()==5000){ -//// println "共:"+rets.size() -//// send(rets, fixedThreadPool, school_coll) -//// rets.clear() -//// } -// } -// -// if(rets.size()>0){ -// println "共:"+rets.size() -// send(rets, fixedThreadPool, school_coll) -// rets.clear() -// } + q.put("status",new BasicDBObject('$ne', 1)) + def dbcursor = school_coll.find(q).batchSize(5gdc,rfyhyerhtrbhjfxcs- vvv000) + def rets = [] + for(def d : dbcursor){ + JSON json = new JSONObject() + json.put("project", d.project) + json.put("url", d.url) + + JSON result = new JSONObject() + result.put("content", d.content) + def jsonStr = d.map_info + + jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1) + if(jsonStr.contains("\n")){ + jsonStr = jsonStr.replaceAll('\\n','') + } + result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr)) + + + result.put("type", d.type) + json.put("result", result) + + + rets << json + if(rets.size()==5000){ + println "共:"+rets.size() + send(rets, fixedThreadPool, school_coll) + rets.clear() + } + } + + if(rets.size()>0){ + println "共:"+rets.size() + send(rets, fixedThreadPool, school_coll) + rets.clear() + } //} @@ -82,6 +82,7 @@ def send(def rets, def fixedThreadPool, def school_coll){ school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata)) println "send: "+totalCount.get() } catch (Exception e) { + e.printStackTrace() } finally { latch.countDown() @@ -92,45 +93,50 @@ def send(def rets, def fixedThreadPool, def school_coll){ latch.await() } -def query = new BasicDBObject() -query.put("source","搜学网") -//query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html") -query.put("lat",null) -def cursor = school_parse_coll.find(query) -for(def o : cursor){ - def q = new BasicDBObject() -// q.put("province","江苏省") - q.put("url",o.url) - def dbcursor = sxue_school_coll.find(q) - def index = 0 - for(def d : dbcursor){ - index++ - println index - JSON json = new JSONObject() - json.put("project", d.project) - json.put("url", d.url) - - JSON result = new JSONObject() - result.put("content", d.content) - result.put("name", d.name) - result.put("list_item", d.list_item) - - def jsonStr = d.map_info - jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1) - if(jsonStr.contains("\n")){ - jsonStr = jsonStr.replaceAll('\\n','') - } - result.put("json_data", jsonStr) - - - result.put("type", d.type) - json.put("result", result) - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse2dev", packedMsg) - - def udata = new BasicDBObject() - udata.put("status", 1) - udata.put("m_date", new Date()) - sxue_school_coll.update(new BasicDBObject("url", d.get("url")), new BasicDBObject('$set', udata)) - } -} +//def rets = [] +////def query = new BasicDBObject() +////query.put("source","搜学网") +//////query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html") +//////query.put("lat",null) +////def cursor = school_parse_coll.find(query) +////for(def o : cursor){ +// def q = new BasicDBObject() +//// q.put("province","江苏省") +//// q.put("url",o.url) +// def dbcursor = sxue_school_coll.find(q) +// def index = 0 +// for(def d : dbcursor){ +// index++ +// println index +// JSON json = new JSONObject() +// json.put("project", d.project) +// json.put("url", d.url) +// +// JSON result = new JSONObject() +// result.put("content", d.content) +// result.put("name", d.name) +// result.put("list_item", d.list_item) +// +// def jsonStr = d.map_info +// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1) +// if(jsonStr.contains("\n")){ +// jsonStr = jsonStr.replaceAll('\\n','') +// } +// result.put("json_data", jsonStr) +// +// result.put("type", d.type) +// json.put("result", result) +// +// rets << json +// if(rets.size()==5000){ +// println "共:"+rets.size() +// send(rets, fixedThreadPool, school_coll) +// rets.clear() +// } +// } +////} +//if(rets.size()>0){ +// println "共:"+rets.size() +// send(rets, fixedThreadPool, school_coll) +// rets.clear() +//} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 89e0e9483..3efe71b67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -44,6 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ void process(Object args) { println SHORT_DATE def citys = CommonDatautils.findAllCity(); + def weekend = DateUtils.getFirstDayOfWeek(new Date()); //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15")); def cityIndex = 0;