+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220718, 20220819);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+17
-1
@@ -26,7 +26,8 @@ class FjwPricePkCcbPrice {
|
||||
static main(args) {
|
||||
def task = new FjwPricePkCcbPrice()
|
||||
// task.run()
|
||||
task.updatePrice()
|
||||
// task.updatePrice()
|
||||
task.addFjwPrice()
|
||||
}
|
||||
|
||||
def updatePrice(){
|
||||
@@ -41,6 +42,21 @@ class FjwPricePkCcbPrice {
|
||||
})
|
||||
}
|
||||
|
||||
def addFjwPrice(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d->
|
||||
if(!d['M']){
|
||||
def dist = CommonDatautils.findDistrictById(d['D'])
|
||||
if(dist){
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
d['M'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
def ccbPrices = getCommunityBasePriceGj(202208)
|
||||
println "ccb总量:"+ccbPrices.size()
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202207//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202205
|
||||
def endDate = 202207
|
||||
def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202206
|
||||
def endDate = 202208
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+1
-2
@@ -3166,8 +3166,7 @@ class CommonDatautils {
|
||||
DBObject dbo;
|
||||
def prices = []
|
||||
def index = 0
|
||||
while(cur.hasNext()) {
|
||||
def obj = cur.next()
|
||||
for(def obj : cur) {
|
||||
if(index == 0){
|
||||
dbo = obj
|
||||
}
|
||||
|
||||
+1
-1
@@ -149,7 +149,7 @@ class FzDataResultStat {
|
||||
|
||||
|
||||
def exportRegion(city, months){
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","闽侯","福清","连江","闽清","罗源","永泰","平潭"];
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
def datas =[]
|
||||
def header = ["城市", "区域"]
|
||||
for(def m : months){
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202207.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202208.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+6
-5
@@ -1,4 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
package com.fangjia.script.gj.commStat
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
@@ -37,11 +37,12 @@ class CityStat {
|
||||
}
|
||||
|
||||
def dealPriceStat(){
|
||||
def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
|
||||
//def citys = ["上海","北京","广州","合肥","南京","深圳","武汉","郑州","重庆","福州","厦门","青岛","太原","长春","苏州","南宁","沈阳","天津","西安","长沙","大连","南昌","东莞","佛山","济南","南通","镇江","宁波","烟台","哈尔滨"]
|
||||
def citys = ["石家庄","贵阳"]
|
||||
def rets = []
|
||||
rets << ["城市","2021-1月总量"]
|
||||
def startDate = DateUtils.parseFromTightByPattern('2021-12-01','yyyy-MM-dd')
|
||||
def endDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
|
||||
rets << ["城市","2022-4月总量"]
|
||||
def startDate = DateUtils.parseFromTightByPattern('2022-01-01','yyyy-MM-dd')
|
||||
def endDate = DateUtils.parseFromTightByPattern('2022-02-01','yyyy-MM-dd')
|
||||
for(def c : citys){
|
||||
def ret = []
|
||||
ret << c
|
||||
+34
-14
@@ -1,4 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
package com.fangjia.script.gj.commStat
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
@@ -29,24 +29,24 @@ DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "ave
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
|
||||
|
||||
|
||||
def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
|
||||
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
|
||||
//for(def city : citys){
|
||||
//
|
||||
//}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
|
||||
// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
|
||||
// if(keys){
|
||||
// d['G'] = keys.size()
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\江苏\\新疆数据统计.xlsx",{d->
|
||||
//// def keys = getDealPrices(d['A'], DEAL_PRICE_COLL)
|
||||
//// if(keys){
|
||||
//// d['G'] = keys.size()
|
||||
//// }
|
||||
//
|
||||
//// def rets = findHousesByCity(d['A'], "sell", 20210701)
|
||||
// if(d['A']){
|
||||
// def count = findHouseCount(d['A'])
|
||||
// d['H'] = count
|
||||
// }
|
||||
|
||||
// def rets = findHousesByCity(d['A'], "sell", 20210701)
|
||||
if(d['A']){
|
||||
def count = findHouseCount(d['A'])
|
||||
d['H'] = count
|
||||
}
|
||||
|
||||
})
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
def findHouseCount(city){
|
||||
@@ -306,3 +306,23 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
|
||||
// rows << [c, districts.size(), alias_count, addr_alias_count]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
|
||||
|
||||
//区域级统计
|
||||
def city = "合肥"
|
||||
def regions = CommonDatautils.getCityRegions(city)
|
||||
def rows = []
|
||||
for(def region : regions){
|
||||
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
|
||||
def alias_count = 0
|
||||
def addr_alias_count = 0
|
||||
for(def r : districts){
|
||||
if(r.alias && r.alias.size()>0){
|
||||
alias_count ++
|
||||
}
|
||||
if(r.addr_alias && r.addr_alias.size()>0){
|
||||
addr_alias_count++
|
||||
}
|
||||
}
|
||||
rows << [region, districts.size(), alias_count, addr_alias_count]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
package com.fangjia.script.gj.dongguanbank
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class DongGuanBankTest {
|
||||
static main(args) {
|
||||
def task = new DongGuanBankTest()
|
||||
def path = "D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx"
|
||||
// task.run(path)
|
||||
task.addPrice(path)
|
||||
}
|
||||
|
||||
def run(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
def city = d['A'].replace("市","")
|
||||
if(city){
|
||||
def region = RegionData.formatRegion(city, d['B'])
|
||||
def results = OpenApiClient.addressMatch(city, d['F'], "property", false, region, d['C'])
|
||||
if(results && results.precise>=0){
|
||||
d['H'] = results.city
|
||||
d['I'] = results.district
|
||||
d['J'] = results.name
|
||||
d['Q'] = results.propertytype
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\东莞银行\\东莞银行房价数据测试20220916 v2.xlsx", { d ->
|
||||
if(d['H']){
|
||||
def dist = CommonDatautils.findDistrict(d['H'], d['I'], d['J'], "district")
|
||||
def priceObj = CommonDatautils.queryDistrictPrice(d['H'], d['I'], d['J'], DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
|
||||
if(priceObj){
|
||||
d['M'] = priceObj.avg_price
|
||||
d['N'] = (priceObj.avg_price * d['G']).toInteger()
|
||||
d['O'] = ((int)(priceObj.avg_price/100))*100
|
||||
if(dist){
|
||||
d['R'] = dist.estate_type2
|
||||
}
|
||||
if(priceObj.value_type){
|
||||
if(priceObj.value_type == 1){
|
||||
d['S'] = "指导价"
|
||||
}else if(priceObj.value_type == 2){
|
||||
d['S'] = "自身样本"
|
||||
}else if(priceObj.value_type == 3){
|
||||
d['S'] = "第三方"
|
||||
}else if(priceObj.value_type == 4){
|
||||
d['S'] = "交易数据"
|
||||
}else if(priceObj.value_type == 5){
|
||||
d['S'] = "周边楼盘"
|
||||
}else if(priceObj.value_type == 6){
|
||||
d['S'] = "推导"
|
||||
}else if(priceObj.value_type == 9){
|
||||
d['S'] = "神经网络"
|
||||
}else if(priceObj.value_type == 10){
|
||||
d['S'] = "无方式"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!d['P']){
|
||||
def alias = []
|
||||
if(!dist.alias){
|
||||
alias = []
|
||||
}else{
|
||||
alias = dist.alias
|
||||
}
|
||||
alias << dist.name
|
||||
def map = findMaxAvgPrice(dist.city, dist.region, alias)
|
||||
if(map){
|
||||
d['P'] = map.max
|
||||
d['Q'] = map.min
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findMaxAvgPrice(def city, def region, def alias){
|
||||
def set = null
|
||||
try{
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= 20210101 and region='${region}' and district_name in ('${aliasStr}')"
|
||||
|
||||
set = new HashSet()
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set << data.avg_price
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
if(set && set.size()>=3){
|
||||
def map = [:]
|
||||
set = set.sort()
|
||||
map.min = set[0]
|
||||
map.max = set[set.size()-1]
|
||||
return map
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def findMinPrice(city){
|
||||
def avg_price = null
|
||||
ElasticUtils.executeSqlQuery("select min(avg_price) as avg_price from seawater where p_date >= 20210901 and city='${city}'", {
|
||||
avg_price = it.avg_price
|
||||
})
|
||||
return avg_price
|
||||
}
|
||||
|
||||
}
|
||||
+24
-10
@@ -104,9 +104,23 @@ public class BeiKeCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
for(def m : citys){
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
println m.city +" 详情链接:"+detailUrls.size()
|
||||
driver.get(m.url+"/chengjiao/")
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
def regionText = r.text()
|
||||
def href = m.url+ r.attr("href")
|
||||
|
||||
println "开始抓取【${m.city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, m.city, href, regionText)
|
||||
println m.url +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
|
||||
// def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
// println m.city +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
@@ -142,12 +156,12 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def getPageDeail(driver, city, cityUrl, region){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "/chengjiao/pg${page}/"
|
||||
def url = cityUrl + "pg${page}/"
|
||||
try{
|
||||
driver.get(url)
|
||||
}catch(Exception e){
|
||||
@@ -164,7 +178,7 @@ public class BeiKeCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
parse(city, e, region)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
@@ -182,7 +196,7 @@ public class BeiKeCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
def parse(city,e, region){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
@@ -190,7 +204,7 @@ public class BeiKeCrawlByChrome {
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
detail.region = region//findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
@@ -376,13 +390,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("福州")){
|
||||
// if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
// cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+2
-2
@@ -177,7 +177,7 @@ class ExportPaimai {
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "南京"
|
||||
def city = "合肥"
|
||||
def region = null
|
||||
def datas = findPaimaiData(city,region)
|
||||
println "总量:"+datas.size()
|
||||
@@ -241,7 +241,7 @@ class ExportPaimai {
|
||||
println d.title +" "+d.url
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\南京法拍.xlsx",rows,"数据")
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\法拍\\合肥法拍.xlsx",rows,"数据")
|
||||
|
||||
// batchCreateFile(rows, "D:\\数据\\重庆农商行\\法拍\\淘宝拍卖(新).csv",null,"GBK")
|
||||
println "生成完成........"
|
||||
|
||||
+5
-5
@@ -34,9 +34,9 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.addDealRegion();
|
||||
task.standardRegion()
|
||||
// task.updadteDealPriceField()
|
||||
}
|
||||
@@ -152,7 +152,7 @@ public class LianjiaCrawlByChrome {
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "福州")
|
||||
// q.put("city", "福州")
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
@@ -250,7 +250,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-01", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e, regionText)
|
||||
@@ -469,7 +469,7 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["福州"]
|
||||
def citys= ["石家庄","贵阳"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
|
||||
+20
-14
@@ -66,8 +66,8 @@ class PaimaiDataSend {
|
||||
def rmfysszcRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
|
||||
q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
|
||||
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市", "苏州市"]))
|
||||
// q.put("url", "https://www.rmfysszc.gov.cn/statichtml/rm_obj/88061.shtml")
|
||||
def dbcursor = rmfysszc_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
@@ -149,9 +149,9 @@ class PaimaiDataSend {
|
||||
def jdRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市"]))
|
||||
q.put("city",new BasicDBObject('$in', ["南京市", "宿迁市","苏州市"]))
|
||||
// q.put("parser_status", new BasicDBObject('$ne',3))
|
||||
q.put("url","https://paimai.jd.com/115669796")
|
||||
// q.put("url","https://paimai.jd.com/115669796")
|
||||
def dbcursor = jd_coll.find(q).batchSize(3000)
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
@@ -187,19 +187,20 @@ class PaimaiDataSend {
|
||||
|
||||
def aliZcRun(){
|
||||
def pm_q = new BasicDBObject()
|
||||
pm_q.put("city",new BasicDBObject('$in', ["南京", "宿迁","苏州"]))
|
||||
// pm_q.put("city","苏州")
|
||||
// pm_q.put("region","姑苏")
|
||||
pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
|
||||
// pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")))
|
||||
// Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
// pm_q.put("url", new BasicDBObject('$regex', pattern))
|
||||
// pm_q.put("url","https://sf-item.taobao.com/sf_item/659117468505.htm")
|
||||
pm_q.put("mDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-08-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2022-09-15")))
|
||||
Pattern pattern = Pattern.compile("taobao.com", Pattern.MULTILINE);
|
||||
pm_q.put("url", new BasicDBObject('$regex', pattern))
|
||||
// pm_q.put("mappingEstateType", null)
|
||||
// pm_q.put("area", new BasicDBObject('$lt', 30))
|
||||
def dbcursor = paimai43_COLL.find(pm_q).batchSize(1000)
|
||||
println dbcursor.count()
|
||||
for(def d : dbcursor){
|
||||
// def flag = sendAliZc(d.url)
|
||||
def flag = sendAli(d.url)
|
||||
def flag = sendAliZc(d.url)
|
||||
// def flag = sendAli(d.url)
|
||||
if(!flag){
|
||||
println "不存在:"+d.url
|
||||
}
|
||||
@@ -240,7 +241,6 @@ class PaimaiDataSend {
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("parser_status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
@@ -294,7 +294,7 @@ class PaimaiDataSend {
|
||||
def aliFpRun(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["南京","宿迁"]))
|
||||
q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
// q.put("parser_status", new BasicDBObject('$ne',2))
|
||||
// q.put("url","https://sf-item.taobao.com/sf_item/631662596036.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
@@ -373,10 +373,11 @@ class PaimaiDataSend {
|
||||
def aliZcRun2(){
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",new BasicDBObject('$in', ["上海"]))
|
||||
// q.put("city",new BasicDBObject('$in', ["苏州","宿迁","南京"]))
|
||||
q.put("city","上海")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',1))
|
||||
// q.put("region","姑苏")
|
||||
// q.put("url","https://sf-item.taobao.com/sf_item/586415617368.htm")
|
||||
q.put("url","https://susong-item.taobao.com/auction/652481594083.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
@@ -541,6 +542,11 @@ class PaimaiDataSend {
|
||||
o.mappingAgencyName = d.mappingAgencyName//映射资产类型
|
||||
o.mappingStatus = d.mappingStatus//映射拍卖状态
|
||||
o.mappingStage = d.mappingStage
|
||||
|
||||
if(d.startRateDiscount){
|
||||
o.startRateDiscount = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, d.startRateDiscount);//起拍折价率 起拍折价率 = 起拍价 / 市场价或评估价
|
||||
}
|
||||
o.diffPrice = d.diffPrice;//差价 差价 = 成交价-市场价或评估价
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,19 +16,152 @@ import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "192.168.0.235", 27018)
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("root", "anjuke.com")
|
||||
def dbcursor = coll.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
map.put(d.city, d.city_short)
|
||||
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
|
||||
// def city = RegionData.formatCity(d['E'])
|
||||
// if(city && !d['J']){
|
||||
// def region = RegionData.formatRegion(city, d['G'])
|
||||
// d['I'] = city
|
||||
// d['J'] = region
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx",{d->
|
||||
// def city = RegionData.formatCity(d['D'])
|
||||
// if(!city){
|
||||
// city = RegionData.formatCity(d['F'])
|
||||
// }
|
||||
// if(city && !d['H']){
|
||||
// def region = RegionData.formatRegion(city, d['B'])
|
||||
// d['G'] = city
|
||||
// d['H'] = region
|
||||
// }
|
||||
//})
|
||||
|
||||
//def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
//def datas = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
|
||||
|
||||
//def fjwCitys = []
|
||||
//for(def d : datas){
|
||||
// fjwCitys << d['G']
|
||||
//}
|
||||
//def rows = []
|
||||
//for(def d : datas){
|
||||
// if(citys.contains(d['G'])){
|
||||
// rows << d
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\建行\\二期\\统计局行政区划与房价网区域比对V2.xlsx", rows, "区域")
|
||||
|
||||
//for(def c : citys){
|
||||
// if(!fjwCitys.contains(c)){
|
||||
// println c
|
||||
// }
|
||||
//}
|
||||
|
||||
//
|
||||
//ExcelUtils.write("D:\\数据\\建行\\二期\\(特殊城市)统计局行政区划与房价网区域比对.xlsx", rows, "区域")
|
||||
//
|
||||
|
||||
//def map = [:]
|
||||
//def datas1 = ExcelUtils.read("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx")
|
||||
//for(def row : datas1){
|
||||
// if(row['J']){
|
||||
// def codeMap = [:]
|
||||
// codeMap.prov = row['B']
|
||||
// codeMap.city = row['F']
|
||||
// codeMap.region = row['H']
|
||||
// map.put(row['I']+"#"+row['J'], codeMap)
|
||||
// }
|
||||
//}
|
||||
//def datas2 = ExcelUtils.read("D:\\数据\\建行\\二期\\统计局行政区划信息.xlsx")
|
||||
//for(def row : datas2){
|
||||
// if(row['H']){
|
||||
// def codeMap = [:]
|
||||
// codeMap.prov = row['E']
|
||||
// codeMap.city = row['C']
|
||||
// codeMap.region = row['A']
|
||||
// map.put(row['G']+"#"+row['H'], codeMap)
|
||||
// }
|
||||
//}
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
|
||||
// def key = d['B'] +"#"+d['C']
|
||||
// def codeMap = map.get(key)
|
||||
// if(codeMap && !d['D']){
|
||||
// d['D'] = codeMap.prov
|
||||
// d['E'] = codeMap.city
|
||||
// d['F'] = codeMap.region
|
||||
// }
|
||||
//})
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
|
||||
if(!d['D']){
|
||||
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
|
||||
if(infos && infos.size()==1){
|
||||
def info = infos[0]
|
||||
d['D'] = info.province_code
|
||||
d['E'] = info.area_code
|
||||
d['F'] = info.street_code
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
def findTownshipInfo(city, region, coll){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
|
||||
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
|
||||
|
||||
def dbcursor = coll.find(q)
|
||||
def rows = []
|
||||
for(def d : dbcursor){
|
||||
// if(d.street_name.contains(region)){
|
||||
rows << d
|
||||
// }
|
||||
}
|
||||
return rows
|
||||
}
|
||||
println map.size()
|
||||
println map
|
||||
|
||||
|
||||
|
||||
//def cityMap = CommonDatautils.findAllCityProvince()
|
||||
//def rows = []
|
||||
//rows << ["省", "市", "区域"]
|
||||
//cityMap.each{c,p->
|
||||
// def regions = CommonDatautils.getCityRegions(c)
|
||||
// for(def r : regions){
|
||||
// def row = []
|
||||
// row << p
|
||||
// row << c
|
||||
// row << r
|
||||
// rows << row
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\亲邻\\房价网省市区.xlsx", rows, "区域")
|
||||
|
||||
|
||||
|
||||
|
||||
//def q = new BasicDBObject()
|
||||
//q.put("root", "anjuke.com")
|
||||
//def dbcursor = coll.find(q)
|
||||
//def map = [:]
|
||||
//for(def d : dbcursor){
|
||||
// map.put(d.city, d.city_short)
|
||||
//}
|
||||
//println map.size()
|
||||
//println map
|
||||
|
||||
|
||||
//def dist = CommonDatautils.findDistrictById("62ce233a3705890001dfa164")
|
||||
//DBUtils.seaweed.save(dist)
|
||||
//DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId("62ce233a3705890001dfa164")), new BasicDBObject('$set', dist))
|
||||
|
||||
+87
-81
@@ -19,47 +19,47 @@ DBCollection sxue_school_coll = MongoUtils.getCollectionByName("crawl_tmp", "sch
|
||||
|
||||
DBCollection school_parse_coll = MongoUtils.getCollectionByName("fangjia_craw", "school", "192.168.0.235", 27018)
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
//Thread.sleep(60*60*1000)
|
||||
//while (true){
|
||||
// def q = new BasicDBObject()
|
||||
def q = new BasicDBObject()
|
||||
// q.put("province",new BasicDBObject('$in', ["江苏","江苏省"]))
|
||||
// q.put("status",new BasicDBObject('$ne', 1))
|
||||
// def dbcursor = school_coll.find(q).limit(1000)
|
||||
// def rets = []
|
||||
// for(def d : dbcursor){
|
||||
// JSON json = new JSONObject()
|
||||
// json.put("project", d.project)
|
||||
// json.put("url", d.url)
|
||||
//
|
||||
// JSON result = new JSONObject()
|
||||
// result.put("content", d.content)
|
||||
// def jsonStr = d.map_info
|
||||
//
|
||||
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
|
||||
// if(jsonStr.contains("\n")){
|
||||
// jsonStr = jsonStr.replaceAll('\\n','')
|
||||
// }
|
||||
// result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
|
||||
//
|
||||
//
|
||||
// result.put("type", d.type)
|
||||
// json.put("result", result)
|
||||
//
|
||||
//
|
||||
// rets << json
|
||||
//// if(rets.size()==5000){
|
||||
//// println "共:"+rets.size()
|
||||
//// send(rets, fixedThreadPool, school_coll)
|
||||
//// rets.clear()
|
||||
//// }
|
||||
// }
|
||||
//
|
||||
// if(rets.size()>0){
|
||||
// println "共:"+rets.size()
|
||||
// send(rets, fixedThreadPool, school_coll)
|
||||
// rets.clear()
|
||||
// }
|
||||
q.put("status",new BasicDBObject('$ne', 1))
|
||||
def dbcursor = school_coll.find(q).batchSize(5gdc,rfyhyerhtrbhjfxcs- vvv000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", d.project)
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
def jsonStr = d.map_info
|
||||
|
||||
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
|
||||
if(jsonStr.contains("\n")){
|
||||
jsonStr = jsonStr.replaceAll('\\n','')
|
||||
}
|
||||
result.put("data_json", net.sf.json.JSONObject.fromObject(jsonStr))
|
||||
|
||||
|
||||
result.put("type", d.type)
|
||||
json.put("result", result)
|
||||
|
||||
|
||||
rets << json
|
||||
if(rets.size()==5000){
|
||||
println "共:"+rets.size()
|
||||
send(rets, fixedThreadPool, school_coll)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
println "共:"+rets.size()
|
||||
send(rets, fixedThreadPool, school_coll)
|
||||
rets.clear()
|
||||
}
|
||||
//}
|
||||
|
||||
|
||||
@@ -82,6 +82,7 @@ def send(def rets, def fixedThreadPool, def school_coll){
|
||||
school_coll.update(new BasicDBObject("url", json.get("url")), new BasicDBObject('$set', udata))
|
||||
println "send: "+totalCount.get()
|
||||
} catch (Exception e) {
|
||||
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
@@ -92,45 +93,50 @@ def send(def rets, def fixedThreadPool, def school_coll){
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("source","搜学网")
|
||||
//query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
|
||||
query.put("lat",null)
|
||||
def cursor = school_parse_coll.find(query)
|
||||
for(def o : cursor){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("province","江苏省")
|
||||
q.put("url",o.url)
|
||||
def dbcursor = sxue_school_coll.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", d.project)
|
||||
json.put("url", d.url)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", d.content)
|
||||
result.put("name", d.name)
|
||||
result.put("list_item", d.list_item)
|
||||
|
||||
def jsonStr = d.map_info
|
||||
jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
|
||||
if(jsonStr.contains("\n")){
|
||||
jsonStr = jsonStr.replaceAll('\\n','')
|
||||
}
|
||||
result.put("json_data", jsonStr)
|
||||
|
||||
|
||||
result.put("type", d.type)
|
||||
json.put("result", result)
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
sxue_school_coll.update(new BasicDBObject("url", d.get("url")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
//def rets = []
|
||||
////def query = new BasicDBObject()
|
||||
////query.put("source","搜学网")
|
||||
//////query.put("url","http://xuexiao.51sxue.com/detail/id_26799.html")
|
||||
//////query.put("lat",null)
|
||||
////def cursor = school_parse_coll.find(query)
|
||||
////for(def o : cursor){
|
||||
// def q = new BasicDBObject()
|
||||
//// q.put("province","江苏省")
|
||||
//// q.put("url",o.url)
|
||||
// def dbcursor = sxue_school_coll.find(q)
|
||||
// def index = 0
|
||||
// for(def d : dbcursor){
|
||||
// index++
|
||||
// println index
|
||||
// JSON json = new JSONObject()
|
||||
// json.put("project", d.project)
|
||||
// json.put("url", d.url)
|
||||
//
|
||||
// JSON result = new JSONObject()
|
||||
// result.put("content", d.content)
|
||||
// result.put("name", d.name)
|
||||
// result.put("list_item", d.list_item)
|
||||
//
|
||||
// def jsonStr = d.map_info
|
||||
// jsonStr = jsonStr.substring(jsonStr.indexOf("{"), jsonStr.toString().lastIndexOf("}")+1)
|
||||
// if(jsonStr.contains("\n")){
|
||||
// jsonStr = jsonStr.replaceAll('\\n','')
|
||||
// }
|
||||
// result.put("json_data", jsonStr)
|
||||
//
|
||||
// result.put("type", d.type)
|
||||
// json.put("result", result)
|
||||
//
|
||||
// rets << json
|
||||
// if(rets.size()==5000){
|
||||
// println "共:"+rets.size()
|
||||
// send(rets, fixedThreadPool, school_coll)
|
||||
// rets.clear()
|
||||
// }
|
||||
// }
|
||||
////}
|
||||
//if(rets.size()>0){
|
||||
// println "共:"+rets.size()
|
||||
// send(rets, fixedThreadPool, school_coll)
|
||||
// rets.clear()
|
||||
//}
|
||||
+1
@@ -44,6 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
void process(Object args) {
|
||||
println SHORT_DATE
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
|
||||
def cityIndex = 0;
|
||||
|
||||
Reference in New Issue
Block a user