Merge remote-tracking branch 'origin/script_gj' into script_gj

This commit is contained in:
Jian Shen
2022-09-19 17:40:03 +08:00
16 changed files with 479 additions and 148 deletions
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220718, 20220819);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -26,7 +26,8 @@ class FjwPricePkCcbPrice {
static main(args) {
def task = new FjwPricePkCcbPrice()
// task.run()
task.updatePrice()
// task.updatePrice()
task.addFjwPrice()
}
def updatePrice(){
@@ -41,6 +42,21 @@ class FjwPricePkCcbPrice {
})
}
def addFjwPrice(){
ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d->
if(!d['M']){
def dist = CommonDatautils.findDistrictById(d['D'])
if(dist){
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
if(dbo){
d['M'] = dbo.avg_price
}
}
}
})
}
def run(){
def ccbPrices = getCommunityBasePriceGj(202208)
println "ccb总量:"+ccbPrices.size()
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202207//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202205
def endDate = 202207
def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202206
def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -3166,8 +3166,7 @@ class CommonDatautils {
DBObject dbo;
def prices = []
def index = 0
while(cur.hasNext()) {
def obj = cur.next()
for(def obj : cur) {
if(index == 0){
dbo = obj
}
@@ -149,7 +149,7 @@ class FzDataResultStat {
def exportRegion(city, months){
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","闽侯","福清","连江","闽清","罗源","永泰","平潭"];
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
def datas =[]
def header = ["城市", "区域"]
for(def m : months){
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202207.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -1,4 +1,4 @@
package com.fangjia.script.gj.test
package com.fangjia.script.gj.commStat
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
@@ -37,11 +37,12 @@ class CityStat {
}
def dealPriceStat(){
def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
//def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
def citys = ["石家庄","贵阳"]
def rets = []
rets << ["城市","2021-1月总量"]
def startDate = DateUtils.parseFromTightByPattern('2021-12-01','yyyy-MM-dd')
def endDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
rets << ["城市","2022-4月总量"]
def startDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
def endDate = DateUtils.parseFromTightByPattern('2022-02-01','yyyy-MM-dd')
for(def c : citys){
def ret = []
ret << c
@@ -1,4 +1,4 @@
package com.fangjia.script.gj.test
package com.fangjia.script.gj.commStat
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
@@ -29,24 +29,24 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
//for(def city : citys){
//
//}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
// if(keys){
// d['G'] = keys.size()
//ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
//// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
//// if(keys){
//// d['G'] = keys.size()
//// }
//
//// def rets = findHousesByCity(d['A'], "sell", 20210701)
// if(d['A']){
// def count = findHouseCount(d['A'])
// d['H'] = count
// }
// def rets = findHousesByCity(d['A'], "sell", 20210701)
if(d['A']){
def count = findHouseCount(d['A'])
d['H'] = count
}
})
//
//})
def findHouseCount(city){
@@ -306,3 +306,23 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
// rows << [c, districts.size(), alias_count, addr_alias_count]
//}
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
//区域级统计
def city = "合肥"
def regions = CommonDatautils.getCityRegions(city)
def rows = []
for(def region : regions){
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
def alias_count = 0
def addr_alias_count = 0
for(def r : districts){
if(r.alias && r.alias.size()>0){
alias_count ++
}
if(r.addr_alias && r.addr_alias.size()>0){
addr_alias_count++
}
}
rows << [region, districts.size(), alias_count, addr_alias_count]
}
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
@@ -0,0 +1,135 @@
package com.fangjia.script.gj.dongguanbank
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import org.apache.lucene.queryparser.classic.QueryParser
class DongGuanBankTest {
static main(args) {
def task = new DongGuanBankTest()
def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx"
// task.run(path)
task.addPrice(path)
}
def run(path){
ExcelUtils.multiThreadingModify(path,{d->
def city = d['A'].replace("市","")
if(city){
def region = RegionData.formatRegion(city, d['B'])
def results = OpenApiClient.addressMatch(city, d['F'], "property", false, region, d['C'])
if(results && results.precise>=0){
d['H'] = results.city
d['I'] = results.district
d['J'] = results.name
d['Q'] = results.propertytype
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify("D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx", { d ->
if(d['H']){
def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district")
def priceObj = CommonDatautils.queryDistrictPrice(d['H'], d['I'], d['J'], DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
if(priceObj){
d['M'] = priceObj.avg_price
d['N'] = (priceObj.avg_price * d['G']).toInteger()
d['O'] = ((int)(priceObj.avg_price/100))*100
if(dist){
d['R'] = dist.estate_type2
}
if(priceObj.value_type){
if(priceObj.value_type == 1){
d['S'] = "指导价"
}else if(priceObj.value_type == 2){
d['S'] = "自身样本"
}else if(priceObj.value_type == 3){
d['S'] = "第三方"
}else if(priceObj.value_type == 4){
d['S'] = "交易数据"
}else if(priceObj.value_type == 5){
d['S'] = "周边楼盘"
}else if(priceObj.value_type == 6){
d['S'] = "推导"
}else if(priceObj.value_type == 9){
d['S'] = "神经网络"
}else if(priceObj.value_type == 10){
d['S'] = "无方式"
}
}
}
if(!d['P']){
def alias = []
if(!dist.alias){
alias = []
}else{
alias = dist.alias
}
alias << dist.name
def map = findMaxAvgPrice(dist.city, dist.region, alias)
if(map){
d['P'] = map.max
d['Q'] = map.min
}
}
}
})
}
def findMaxAvgPrice(def city, def region, def alias){
def set = null
try{
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')"
set = new HashSet()
ElasticUtils.executeSqlQuery(sql,{data->
set << data.avg_price
})
}catch(Exception e){
e.printStackTrace()
}
if(set && set.size()>=3){
def map = [:]
set = set.sort()
map.min = set[0]
map.max = set[set.size()-1]
return map
}
return null
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def findMinPrice(city){
def avg_price = null
ElasticUtils.executeSqlQuery("select min(avg_price) as avg_price from seawater where p_date >= 20210901 and city='${city}'", {
avg_price = it.avg_price
})
return avg_price
}
}
@@ -104,9 +104,23 @@ public class BeiKeCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def m : citys){
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, m.city, m.url)
println m.city +" 详情链接:"+detailUrls.size()
driver.get(m.url+"/chengjiao/")
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
def href = m.url+ r.attr("href")
println "开始抓取【${m.city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, m.city, href, regionText)
println m.url +" 详情链接:"+detailUrls.size()
}
}
// def detailUrls = getPageDeail(driver, m.city, m.url)
// println m.city +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -142,12 +156,12 @@ public class BeiKeCrawlByChrome {
}
def getPageDeail(driver, city, cityUrl){
def getPageDeail(driver, city, cityUrl, region){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "/chengjiao/pg${page}/"
def url = cityUrl + "pg${page}/"
try{
driver.get(url)
}catch(Exception e){
@@ -164,7 +178,7 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
parse(city, e, region)
}else{
stop = true;
break;
@@ -182,7 +196,7 @@ public class BeiKeCrawlByChrome {
return detailUrls
}
def parse(city,e){
def parse(city,e, region){
try {
def title = e.select(".title a").text()
if (title) {
@@ -190,7 +204,7 @@ public class BeiKeCrawlByChrome {
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
@@ -376,13 +390,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("福州")){
// if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
// cityMap.put(e.text(), "https:"+href)
}
// }
}
}
return list
@@ -177,7 +177,7 @@ class ExportPaimai {
def run(){
def city = "南京"
def city = "合肥"
def region = null
def datas = findPaimaiData(city,region)
println "总量:"+datas.size()
@@ -241,7 +241,7 @@ class ExportPaimai {
println d.title +" "+d.url
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\南京法拍.xlsx",rows,"数据")
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据")
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
println "生成完成........"
@@ -34,9 +34,9 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
// task.run()
task.run()
// task.run1()
task.addDealRegion();
// task.addDealRegion();
task.standardRegion()
// task.updadteDealPriceField()
}
@@ -152,7 +152,7 @@ public class LianjiaCrawlByChrome {
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("city", "福州")
// q.put("city", "福州")
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
@@ -250,7 +250,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-01", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
@@ -469,7 +469,7 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["福州"]
def citys= ["石家庄","贵阳"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
@@ -66,8 +66,8 @@ class PaimaiDataSend {
def rmfysszcRun(){
def index = 0
def q = new BasicDBObject()
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -149,9 +149,9 @@ class PaimaiDataSend {
def jdRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
// q.put("parser_status", new BasicDBObject('$ne',3))
q.put("url","https://paimai.jd.com/115669796")
// q.put("url","https://paimai.jd.com/115669796")
def dbcursor = jd_coll.find(q).batchSize(3000)
for(def d : dbcursor){
index++
@@ -187,19 +187,20 @@ class PaimaiDataSend {
def aliZcRun(){
def pm_q = new BasicDBObject()
pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
// pm_q.put("city","苏州")
// pm_q.put("region","姑苏")
pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
// pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
pm_q.put("url", new BasicDBObject('$regex', pattern))
// pm_q.put("mappingEstateType", null)
// pm_q.put("area", new BasicDBObject('$lt', 30))
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
println dbcursor.count()
for(def d : dbcursor){
// def flag = sendAliZc(d.url)
def flag = sendAli(d.url)
def flag = sendAliZc(d.url)
// def flag = sendAli(d.url)
if(!flag){
println "不存在:"+d.url
}
@@ -240,7 +241,6 @@ class PaimaiDataSend {
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("parser_status", 1)
udata.put("m_date", new Date())
@@ -294,7 +294,7 @@ class PaimaiDataSend {
def aliFpRun(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["南京","宿迁"]))
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
// q.put("parser_status", new BasicDBObject('$ne',2))
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
@@ -373,10 +373,11 @@ class PaimaiDataSend {
def aliZcRun2(){
def index = 0
def q = new BasicDBObject()
q.put("city",new BasicDBObject('$in', ["上海"]))
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
q.put("city","上海")
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","姑苏")
// q.put("url","https://sf-item.taobao.com/sf_item/586415617368.htm")
q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
@@ -541,6 +542,11 @@ class PaimaiDataSend {
o.mappingAgencyName = d.mappingAgencyName//映射资产类型
o.mappingStatus = d.mappingStatus//映射拍卖状态
o.mappingStage = d.mappingStage
if(d.startRateDiscount){
o.startRateDiscount = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, d.startRateDiscount);//起拍折价率 起拍折价率 = 起拍价 / 市场价或评估价
}
o.diffPrice = d.diffPrice;//差价 差价 = 成交价-市场价或评估价
rets << o
}
}
@@ -16,19 +16,152 @@ import groovy.sql.Sql
import org.bson.types.ObjectId
import java.text.SimpleDateFormat
import java.util.regex.Pattern
DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018)
def q = new BasicDBObject()
q.put("root", "anjuke.com")
def dbcursor = coll.find(q)
def map = [:]
for(def d : dbcursor){
map.put(d.city, d.city_short)
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
// def city = RegionData.formatCity(d['E'])
// if(city && !d['J']){
// def region = RegionData.formatRegion(city, d['G'])
// d['I'] = city
// d['J'] = region
// }
//})
//ExcelUtils.modify("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx",{d->
// def city = RegionData.formatCity(d['D'])
// if(!city){
// city = RegionData.formatCity(d['F'])
// }
// if(city && !d['H']){
// def region = RegionData.formatRegion(city, d['B'])
// d['G'] = city
// d['H'] = region
// }
//})
//def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
//def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
//def fjwCitys = []
//for(def d : datas){
// fjwCitys << d['G']
//}
//def rows = []
//for(def d : datas){
// if(citys.contains(d['G'])){
// rows << d
// }
//}
//ExcelUtils.write("D:\\数据\\建行\\二期\\统计局行政区划与房价网区域比对V2.xlsx", rows, "区域")
//for(def c : citys){
// if(!fjwCitys.contains(c)){
// println c
// }
//}
//
//ExcelUtils.write("D:\\数据\\建行\\二期\\(特殊城市)统计局行政区划与房价网区域比对.xlsx", rows, "区域")
//
//def map = [:]
//def datas1 = ExcelUtils.read("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx")
//for(def row : datas1){
// if(row['J']){
// def codeMap = [:]
// codeMap.prov = row['B']
// codeMap.city = row['F']
// codeMap.region = row['H']
// map.put(row['I']+"#"+row['J'], codeMap)
// }
//}
//def datas2 = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
//for(def row : datas2){
// if(row['H']){
// def codeMap = [:]
// codeMap.prov = row['E']
// codeMap.city = row['C']
// codeMap.region = row['A']
// map.put(row['G']+"#"+row['H'], codeMap)
// }
//}
//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
// def key = d['B'] +"#"+d['C']
// def codeMap = map.get(key)
// if(codeMap && !d['D']){
// d['D'] = codeMap.prov
// d['E'] = codeMap.city
// d['F'] = codeMap.region
// }
//})
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
if(!d['D']){
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
if(infos && infos.size()==1){
def info = infos[0]
d['D'] = info.province_code
d['E'] = info.area_code
d['F'] = info.street_code
}
}
})
def findTownshipInfo(city, region, coll){
def q = new BasicDBObject()
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
def dbcursor = coll.find(q)
def rows = []
for(def d : dbcursor){
// if(d.street_name.contains(region)){
rows << d
// }
}
return rows
}
println map.size()
println map
//def cityMap = CommonDatautils.findAllCityProvince()
//def rows = []
//rows << ["省", "市", "区域"]
//cityMap.each{c,p->
// def regions = CommonDatautils.getCityRegions(c)
// for(def r : regions){
// def row = []
// row << p
// row << c
// row << r
// rows << row
// }
//}
//ExcelUtils.write("D:\\数据\\亲邻\\房价网省市区.xlsx", rows, "区域")
//def q = new BasicDBObject()
//q.put("root", "anjuke.com")
//def dbcursor = coll.find(q)
//def map = [:]
//for(def d : dbcursor){
// map.put(d.city, d.city_short)
//}
//println map.size()
//println map
//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
//DBUtils.seaweed.save(dist)
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
@@ -19,47 +19,47 @@ DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "sch
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//Thread.sleep(60*60*1000)
//while (true){
// def q = new BasicDBObject()
def q = new BasicDBObject()
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
// q.put("status",new BasicDBObject('$ne', 1))
// def dbcursor = school_coll.find(q).limit(1000)
// def rets = []
// for(def d : dbcursor){
// JSON json = new JSONObject()
// json.put("project", d.project)
// json.put("url", d.url)
//
// JSON result = new JSONObject()
// result.put("content", d.content)
// def jsonStr = d.map_info
//
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
// if(jsonStr.contains("\n")){
// jsonStr = jsonStr.replaceAll('\\n','')
// }
// result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
//
//
// result.put("type", d.type)
// json.put("result", result)
//
//
// rets << json
//// if(rets.size()==5000){
//// println "共:"+rets.size()
//// send(rets, fixedThreadPool, school_coll)
//// rets.clear()
//// }
// }
//
// if(rets.size()>0){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
// }
q.put("status",new BasicDBObject('$ne', 1))
def dbcursor = school_coll.find(q).batchSize(5gdc,rfyhyerhtrbhjfxcs- vvv000)
def rets = []
for(def d : dbcursor){
JSON json = new JSONObject()
json.put("project", d.project)
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
def jsonStr = d.map_info
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
if(jsonStr.contains("\n")){
jsonStr = jsonStr.replaceAll('\\n','')
}
result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
result.put("type", d.type)
json.put("result", result)
rets << json
if(rets.size()==5000){
println "共:"+rets.size()
send(rets, fixedThreadPool, school_coll)
rets.clear()
}
}
if(rets.size()>0){
println "共:"+rets.size()
send(rets, fixedThreadPool, school_coll)
rets.clear()
}
//}
@@ -82,6 +82,7 @@ def send(def rets, def fixedThreadPool, def school_coll){
school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
println "send: "+totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
@@ -92,45 +93,50 @@ def send(def rets, def fixedThreadPool, def school_coll){
latch.await()
}
def query = new BasicDBObject()
query.put("source","搜学网")
//query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
query.put("lat",null)
def cursor = school_parse_coll.find(query)
for(def o : cursor){
def q = new BasicDBObject()
// q.put("province","江苏省")
q.put("url",o.url)
def dbcursor = sxue_school_coll.find(q)
def index = 0
for(def d : dbcursor){
index++
println index
JSON json = new JSONObject()
json.put("project", d.project)
json.put("url", d.url)
JSON result = new JSONObject()
result.put("content", d.content)
result.put("name", d.name)
result.put("list_item", d.list_item)
def jsonStr = d.map_info
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
if(jsonStr.contains("\n")){
jsonStr = jsonStr.replaceAll('\\n','')
}
result.put("json_data", jsonStr)
result.put("type", d.type)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
sxue_school_coll.update(new BasicDBObject("url", d.get("url")), new BasicDBObject('$set', udata))
}
}
//def rets = []
////def query = new BasicDBObject()
////query.put("source","搜学网")
//////query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
//////query.put("lat",null)
////def cursor = school_parse_coll.find(query)
////for(def o : cursor){
// def q = new BasicDBObject()
//// q.put("province","江苏省")
//// q.put("url",o.url)
// def dbcursor = sxue_school_coll.find(q)
// def index = 0
// for(def d : dbcursor){
// index++
// println index
// JSON json = new JSONObject()
// json.put("project", d.project)
// json.put("url", d.url)
//
// JSON result = new JSONObject()
// result.put("content", d.content)
// result.put("name", d.name)
// result.put("list_item", d.list_item)
//
// def jsonStr = d.map_info
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
// if(jsonStr.contains("\n")){
// jsonStr = jsonStr.replaceAll('\\n','')
// }
// result.put("json_data", jsonStr)
//
// result.put("type", d.type)
// json.put("result", result)
//
// rets << json
// if(rets.size()==5000){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
// }
// }
////}
//if(rets.size()>0){
// println "共:"+rets.size()
// send(rets, fixedThreadPool, school_coll)
// rets.clear()
//}
@@ -44,6 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
def cityIndex = 0;