+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-02-1", "2022-03-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202110
|
||||
def endDate = 202112
|
||||
def sDate= 202201//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202111
|
||||
def endDate = 202201
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+8
-8
@@ -39,8 +39,8 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -73,13 +73,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","11月总量","12月总量","1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-10-01"
|
||||
def date = "2021-11-01"
|
||||
def endDate = "2021-12-01"
|
||||
def newDate = "2022-01-01"
|
||||
def startDate = "2021-11-01"
|
||||
def date = "2021-12-01"
|
||||
def endDate = "2022-01-01"
|
||||
def newDate = "2022-02-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -129,7 +129,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-01成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
+82
-1
@@ -12,9 +12,90 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
class DistrictMarge {
|
||||
static main(args) {
|
||||
def task = new DistrictMarge()
|
||||
task.run()
|
||||
//task.run()
|
||||
task.repeatAddress()
|
||||
}
|
||||
|
||||
def repeatAddress(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "lng","address","name_cat"])
|
||||
findRepead(c, districts)
|
||||
}
|
||||
}
|
||||
def findRepead(city, districts){
|
||||
def map = [:]
|
||||
for(def d : districts){
|
||||
if(d.name_cat == 0 && d.lng){
|
||||
if(d.address && d.address.contains("号")){
|
||||
def list = map.get(d.address)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(d.address, list)
|
||||
}
|
||||
list << d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def newMap = [:]
|
||||
map.each {k,vals->
|
||||
if(vals.size()>=2){
|
||||
vals.sort{o1,o2->
|
||||
return o1.name.size() <=> o2.name.size()
|
||||
}
|
||||
newMap.put(k,vals)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
newMap.each {k,vals->
|
||||
def names = []
|
||||
for(def d : vals){
|
||||
def newKey = d.name
|
||||
if(d.name.contains("小区")){
|
||||
newKey = d.name.toString().replace("小区", "")
|
||||
}
|
||||
if(!names.contains(newKey)){
|
||||
names << newKey
|
||||
}else{
|
||||
d.flag = "删除"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId( d._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def values = newMap.values()
|
||||
def allRets = []
|
||||
for(def list : values){
|
||||
allRets.addAll(list)
|
||||
}
|
||||
def results = []
|
||||
def delcount = 0
|
||||
for(def r : allRets){
|
||||
def rows = []
|
||||
rows << r._id.toString()
|
||||
rows << r.city
|
||||
rows << r.name
|
||||
rows << r.address
|
||||
rows << r.name_cat
|
||||
if(r.flag){
|
||||
rows << r.flag
|
||||
delcount++
|
||||
}
|
||||
results << rows
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\小区\\删除重复地址(${city}).xlsx", results, "sheet")
|
||||
println "${city} 共:"+map.size()+" 2个以上:"+newMap.size()+" delcount:"+delcount
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def rets = getDatas()
|
||||
def deleteIndex = 0
|
||||
|
||||
+37
-17
@@ -433,23 +433,43 @@ def city = "
|
||||
// }
|
||||
//})
|
||||
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
|
||||
if (d['B']) {
|
||||
def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
|
||||
if(dist){
|
||||
def alias = dist.mapping_alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(alias && !alias.contains(d['C'])){
|
||||
index++
|
||||
println index
|
||||
alias << d['C']
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mapping_alias",alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
|
||||
// if (d['B']) {
|
||||
// def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(alias && !alias.contains(d['C'])){
|
||||
// index++
|
||||
// println index
|
||||
// alias << d['C']
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
|
||||
if(d['A']){
|
||||
try{
|
||||
def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
|
||||
if(dist){
|
||||
d['B'] = dist.city
|
||||
d['C'] = dist.region
|
||||
d['D'] = dist.name
|
||||
|
||||
def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
|
||||
d['E'] = price
|
||||
}
|
||||
}catch(Exception e){
|
||||
println d['A']
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
+1
-1
@@ -218,7 +218,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","金辉伯爵山")
|
||||
// q.put("name","玉泉路甲46号院")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
|
||||
+4
-4
@@ -23,8 +23,8 @@ class FzDataResultStat {
|
||||
// task.regionPriceTrendRun()
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
def map1 = [:]
|
||||
@@ -53,7 +53,7 @@ class FzDataResultStat {
|
||||
}
|
||||
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
|
||||
}
|
||||
//ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
@@ -94,7 +94,7 @@ class FzDataResultStat {
|
||||
}
|
||||
def prices = exportRegion(city, months)
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
|
||||
ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据")
|
||||
ExcelUtils.write("D:\\数据\\福州\\福州6区均价半年走势_${curMonth}.xlsx", prices, "数据")
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202201.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+5
-2
@@ -143,8 +143,8 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
|
||||
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
|
||||
}
|
||||
if(!retMap.anjuke_esf_price && d.anjuke_esf_price){
|
||||
retMap.anjuke_esf_price = d.anjuke_esf_price
|
||||
if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){
|
||||
retMap.anjuke_esf_price = d.get("58tc_esf_price")
|
||||
}
|
||||
if(!retMap.soufang_esf_price && d.soufang_esf_price){
|
||||
retMap.soufang_esf_price = d.soufang_esf_price
|
||||
@@ -152,6 +152,9 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.lianjia_esf_price && d.lianjia_esf_price){
|
||||
retMap.lianjia_esf_price = d.lianjia_esf_price
|
||||
}
|
||||
if(!retMap.lianjia_esf_price && d.beike_esf_price){
|
||||
retMap.lianjia_esf_price = d.beike_esf_price
|
||||
}
|
||||
|
||||
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
||||
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
||||
|
||||
@@ -45,7 +45,7 @@ class EsHouseStat {
|
||||
|
||||
public static void main(String[] args){
|
||||
def task = new EsHouseStat();
|
||||
//task.run();
|
||||
task.run();
|
||||
// task.testRun("深圳")
|
||||
// task.exportOpenLog();
|
||||
// task.exportMongoLog()
|
||||
@@ -59,7 +59,7 @@ class EsHouseStat {
|
||||
// task.szRentHouseStat();
|
||||
// task.addPrice();
|
||||
|
||||
task.statNoHouseCCBData();
|
||||
// task.statNoHouseCCBData();
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
@@ -337,21 +337,6 @@ class EsHouseStat {
|
||||
return total;
|
||||
}
|
||||
|
||||
def findHouse(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def total = 0;
|
||||
while (rs.next()){
|
||||
total = rs.getInt("count")
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
def findHouseSource(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM \"seawater-20*\" WHERE 1=1")
|
||||
@@ -373,6 +358,21 @@ class EsHouseStat {
|
||||
return statMap;
|
||||
}
|
||||
|
||||
|
||||
def findHouse(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def total = 0;
|
||||
while (rs.next()){
|
||||
total = rs.getInt("count")
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
def testRun(def city){
|
||||
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ class OpenApiUtils {
|
||||
ret.lng2 = gmap.get("lng")
|
||||
ret.lat2 = gmap.get("lat")
|
||||
ret.estateType = ret.type
|
||||
ret.source = "高德"
|
||||
}catch (Exception e){
|
||||
|
||||
}
|
||||
@@ -250,14 +251,14 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
if(type && type.contains("出入口")){
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
|
||||
}else{
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
poiList.add(jo);
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
@@ -302,7 +303,7 @@ class OpenApiUtils {
|
||||
def distname = formatName(ret.get("name"))
|
||||
address = ret.get("address")
|
||||
if((distname && distname.contains(name)) || (distname && name.contains(distname)) || (address && address.contains(name)) || ( address && name.contains(address))){
|
||||
if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
|
||||
// if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
|
||||
lng2 = location.lng
|
||||
lat2 = location.lat
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
|
||||
@@ -311,7 +312,7 @@ class OpenApiUtils {
|
||||
def mapRegion = ret.get("area")
|
||||
distRegion = mapRegion
|
||||
districtName = distname
|
||||
if(address && address.contains(mapRegion)){
|
||||
if(address && mapRegion && address.contains(mapRegion)){
|
||||
address = address.substring(address.indexOf(mapRegion)+mapRegion.size());
|
||||
}
|
||||
if(ret.get("detail_info")){
|
||||
@@ -330,6 +331,7 @@ class OpenApiUtils {
|
||||
map.name = districtName
|
||||
map.region = distRegion
|
||||
map.estateType = estateType
|
||||
map.source = "百度"
|
||||
rets << map
|
||||
|
||||
lat = null
|
||||
@@ -340,12 +342,18 @@ class OpenApiUtils {
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
// }else{
|
||||
// println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
// }
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
|
||||
def score1 = 0
|
||||
if(distname && name){
|
||||
score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
|
||||
}
|
||||
def score2 = 0
|
||||
if(address && name){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
|
||||
}
|
||||
if(score1 || score2){
|
||||
rets << map
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】(score:"+score1+"#"+score2+")"
|
||||
|
||||
@@ -31,7 +31,7 @@ class RegionPointCrawl {
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
|
||||
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
|
||||
+42
-25
@@ -7,13 +7,17 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
/**
|
||||
* 模糊地址经纬度更新
|
||||
@@ -44,13 +48,16 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
//"北京","常州",
|
||||
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
|
||||
def citys =["上海"] //CommonDatautils.findAllCity();
|
||||
|
||||
def citys =["北京"] //CommonDatautils.findAllCity();
|
||||
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
|
||||
//市谊大厦 华三新村 阳光家园
|
||||
for (def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat2", "lat", "lng2", "lng"]);
|
||||
def rets = findVagueAddress(city, districts);
|
||||
findAndUpdateLocation(rets);
|
||||
writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
|
||||
// writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
|
||||
writeCSV(results, "d:\\数据\\结果\\" + city + "地址模糊.csv")
|
||||
// ExcelUtils.write("d:\\数据\\结果\\" + city + "地址模糊.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +65,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
|
||||
def findVagueAddress(def city, def districts) {
|
||||
def rets = []
|
||||
def rows = []
|
||||
for (def district : districts) {
|
||||
def address = district.address;
|
||||
if (address) {
|
||||
@@ -66,10 +74,13 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
//没地址
|
||||
if (!address) {
|
||||
rets << district;
|
||||
|
||||
// rows << [district.city, district.region, district.name, district.address]
|
||||
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
|
||||
} else {
|
||||
if (!findPattern(address)) {
|
||||
rets << district;
|
||||
// rows << [district.city, district.region, district.name, district.address]
|
||||
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
|
||||
} else {
|
||||
println district.city + " " + district.address + " 精确地址"
|
||||
@@ -97,7 +108,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
} else if (address.indexOf("路") == -1 && address.indexOf("街") == -1 && address.indexOf("大道") == -1 && address.indexOf("巷") == -1 && address.indexOf("北里") == -1 && address.indexOf("南里") == -1 && address.indexOf("东里") == -1 && address.indexOf("西里") == -1) {
|
||||
return null;
|
||||
} else {
|
||||
pattern = Pattern.compile("[路|街|大道|巷]\\d+");
|
||||
pattern = Pattern.compile("[路甲|路|街|大道|巷|弄|里]\\d+");
|
||||
matcher = pattern.matcher(address)
|
||||
if(matcher.find()){
|
||||
return s;
|
||||
@@ -123,16 +134,14 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
for (def ret : rets) {
|
||||
index++;
|
||||
|
||||
if(ret.name!="药师新村"){
|
||||
continue
|
||||
}
|
||||
|
||||
def locationMap = findLatAndLng(ret);
|
||||
if (locationMap) {
|
||||
updateLocation(locationMap, ret);
|
||||
} else {
|
||||
println "[没找到] " + ret.city + " " + ret.region + " " + ret.name + " " + ret.address
|
||||
}
|
||||
println "【${index}/${rets.size()}】************************************************************************************"
|
||||
println "【${index}/${rets.size()}】**************【${ret.region}---${ret.name}----${ret.address}】**********************************************************************"
|
||||
|
||||
}
|
||||
}
|
||||
@@ -148,14 +157,14 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
msgMap.db_lng2 = ret.lng2
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
|
||||
if (ret.lat2 != locationMap.lat2) {
|
||||
udata.put('lat2', locationMap.lat2)
|
||||
udata.put('lng2', locationMap.lng2)
|
||||
if (ret.lat2 && ret.lat2 != locationMap.lat2) {
|
||||
msgMap.m_lat2 = locationMap.lat2
|
||||
msgMap.m_lng2 = locationMap.lng2
|
||||
def distance = GeoHelper.GetDistance(ret.lat2, ret.lng2, locationMap.lat2, locationMap.lng2)
|
||||
msgMap.updateLngFlag = true
|
||||
msgMap.distance = distance
|
||||
udata.put('lat2', locationMap.lat2)
|
||||
udata.put('lng2', locationMap.lng2)
|
||||
}
|
||||
|
||||
if (ret.lat != locationMap.lat) {
|
||||
@@ -171,31 +180,39 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
if (udata) {
|
||||
println ret.city + " " + ret.region + " " + ret.name + " [ 修改其经纬度!]"
|
||||
udata.put("m_date", new Date())
|
||||
// DBUtils.seaweed.update(new BasicDBObject('_id',locationMap._id),new BasicDBObject('$set',udata))
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',locationMap.id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
results << msgMap
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def name, def rets) {
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() > 1) {
|
||||
//相似里找有精确地址
|
||||
for (def mapObj : rets) {
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
if (mapObj.address && findPattern(mapObj.address)) {
|
||||
map = mapObj;
|
||||
break;
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name.contains(name) || name.contains(mapObj.name)){
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
if (mapObj.address && findPattern(mapObj.address)) {
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!map){
|
||||
//交带有交叉口地址
|
||||
for (def mapObj : rets) {
|
||||
if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) {
|
||||
if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){
|
||||
map = mapObj;
|
||||
break;
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name.contains(name) || name.contains(mapObj.name)) {
|
||||
if (mapObj.address && (mapObj.address.contains("交汇") || mapObj.address.contains("交叉"))) {
|
||||
if (!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")) {
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,18 +228,18 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
def findLatAndLng(ret) {
|
||||
//https://restapi.amap.com/v3/place/text?keywords=
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(ret.city, ret.region, ret.name)
|
||||
def map = findStandardObject(ret.city, ret.name, rets);
|
||||
def map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
def locationMap = [:]
|
||||
if (!map || !map.lat) {
|
||||
//http://api.map.baidu.com/place/v2/search
|
||||
rets = OpenApiUtils.getBiDuLocation(ret.city, ret.region, ret.name);
|
||||
map = findStandardObject(ret.city, ret.name, rets);
|
||||
map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
}
|
||||
if (!map || !map.lat) {
|
||||
//https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap
|
||||
def code = codeMap.get(ret.city);
|
||||
rets = OpenApiUtils.searchBaiDuMap(ret.name, ret.region, code.toInteger());
|
||||
map = findStandardObject(ret.city, ret.name, rets);
|
||||
map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
}
|
||||
|
||||
// if((!map || !map.lat)){
|
||||
@@ -237,7 +254,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
locationMap.lng2 = (map.lng2.toDouble() * 10000000000).longValue()
|
||||
locationMap.id = ret._id
|
||||
locationMap.city = ret.city
|
||||
if (ret.address.toString().endsWith("路")) {
|
||||
if (map.address) {
|
||||
locationMap.address = map.address.replace("(","").replace(")","");
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -48,7 +48,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-12-01"
|
||||
static def statDate = "2022-01-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年2月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+14
-2
@@ -12,7 +12,19 @@ class TempMatchDistrict {
|
||||
static main(args) {
|
||||
def task = new TempMatchDistrict()
|
||||
// task.run()
|
||||
task.regionConver()
|
||||
// task.regionConver()
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
|
||||
def total = 0
|
||||
for(def city : citys){
|
||||
def count = CommonDatautils.findAllDistrictCount("district", city)
|
||||
total = total + count
|
||||
println city +" "+count
|
||||
}
|
||||
println "共:"+total
|
||||
}
|
||||
|
||||
def regionConver(){
|
||||
@@ -29,7 +41,7 @@ class TempMatchDistrict {
|
||||
}
|
||||
def run(){
|
||||
def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据"
|
||||
def files = ["合肥.xlsx","湖北.xlsx","陕西.xlsx","上海.xlsx","深圳.xlsx","苏州.xlsx","天津.xlsx","雄安.xlsx","浙江.xlsx","重庆.xlsx"]
|
||||
def files = ["杭州.xlsx"]
|
||||
for(def f : files){
|
||||
def filePath = path+"\\"+f;
|
||||
match(filePath);
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ class FangjiaClientUtils {
|
||||
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
||||
JSONObject jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
if(address){
|
||||
url = url + "&address="+address
|
||||
|
||||
+2
-2
@@ -40,9 +40,9 @@ class LocationTest {
|
||||
// System.out.println(str);
|
||||
|
||||
def task = new LocationTest()
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
// task.run3()
|
||||
// task.run4()
|
||||
// task.run5()
|
||||
}
|
||||
|
||||
+16
-12
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
@@ -33,8 +34,8 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
}
|
||||
|
||||
@@ -94,12 +95,12 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run(){
|
||||
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
// System.setProperty("http.nonProxyHosts", "localhost");
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
// def cityMap = getAllCityUrls(driver)
|
||||
// getLogin(driver)
|
||||
// crawlData(driver, cityMap)
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
}
|
||||
@@ -109,7 +110,7 @@ public class LianjiaCrawlByChrome {
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["Á´¼ÒÔÚÏß","±´¿ÇÕÒ·¿"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
@@ -193,7 +194,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -219,6 +220,9 @@ public class LianjiaCrawlByChrome {
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
@@ -391,9 +395,9 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(!href.contains("fang.lianjia")){
|
||||
// if(!href.contains("fang.lianjia") && e.text().contains("¸£ÖÝ")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
+9
-6
@@ -60,12 +60,12 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
q.put("source", "城市房产")
|
||||
q.put("s_date",202012)
|
||||
q.put("city","±±¾©")
|
||||
q.put("source", "ËÑ·¿")
|
||||
q.put("s_date",202112)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = ".cityhouse.cn/ha/"
|
||||
def prefix = "esf.fang.com/loupan/"
|
||||
def keyList =new HashSet()
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
@@ -77,11 +77,14 @@ class CrawlDistrictIdSelect {
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".htm"))
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
|
||||
if(!shortCity){
|
||||
shortCity = "bj"
|
||||
}
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
@@ -92,7 +95,7 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "cityhouse.com"
|
||||
map.root = "fang.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
|
||||
+2
-2
@@ -68,7 +68,7 @@ class FZloationUpdate {
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[ÐÞ¸Ä-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
@@ -153,7 +153,7 @@ class FZloationUpdate {
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
if(reps && reps.count && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "58ͬ³Ç"//
|
||||
def citys = findCrawlCity(source, 202111);
|
||||
def source = "城市房产"//
|
||||
def citys = findCrawlCity(source, 202112);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202111);
|
||||
def count2 = findCount(source, city, 202112);
|
||||
def count1 = findCount(source, city, 202112);
|
||||
def count2 = findCount(source, city, 202201);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+131
-129
File diff suppressed because one or more lines are too long
+9
-1
@@ -315,11 +315,11 @@ class AddMetadataBatchQfang{
|
||||
private static void saveThirdPartyPrice(def saveDate){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", saveDate.get("xqUrl"))
|
||||
q.put("s_date", SHORT_DATE);
|
||||
q.put("city", saveDate.get("city"));
|
||||
q.put("region", saveDate.get("region"));
|
||||
q.put("name", saveDate.get("xqName"));
|
||||
q.put("source", saveDate.get("source"));
|
||||
|
||||
def udata = THIRD_PARTY_PRICE_COLL.findOne(q)
|
||||
if(!udata){
|
||||
@@ -336,6 +336,14 @@ class AddMetadataBatchQfang{
|
||||
obj.put("source", saveDate.get("source"));
|
||||
obj.put("c_date", new Date());
|
||||
THIRD_PARTY_PRICE_COLL.save(obj);
|
||||
println "[save] "+saveDate.toString()
|
||||
}else{
|
||||
def updata = new BasicDBObject()
|
||||
updata.put("avg_price", saveDate.get("price"))
|
||||
updata.put("url", saveDate.get("xqUrl"));
|
||||
updata.put("m_date", new Date());
|
||||
THIRD_PARTY_PRICE_COLL.update(q,new BasicDBObject('$set',updata))
|
||||
println "[update] "+saveDate.toString()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -87,7 +87,7 @@ class AddSeedBatchQfang extends BaseBatch{
|
||||
href = "https:"+href;
|
||||
}
|
||||
if(href&&!CITYS.contains(city)){
|
||||
ret.put(city, href);
|
||||
ret.put(city, href);
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
|
||||
+9
-4
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["fang.com":"搜房"]
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -94,11 +94,16 @@ class SynchDistrictPrice {
|
||||
for(def r : rets){
|
||||
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(obj){
|
||||
updateIndex++
|
||||
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}else{
|
||||
thirdPartyColl.save(r)
|
||||
try{
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
// if(!obj){
|
||||
// addIndex++
|
||||
@@ -112,7 +117,7 @@ class SynchDistrictPrice {
|
||||
// }
|
||||
// }
|
||||
}
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 已存在:"+updateIndex
|
||||
}
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202111);
|
||||
def count2 = findTotalCount(source, 202112);
|
||||
def count1 = findTotalCount(source, 202112);
|
||||
def count2 = findTotalCount(source, 202201);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20211201", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220101", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20220101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220201", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
Reference in New Issue
Block a user