This commit is contained in:
2022-05-06 15:28:55 +08:00
parent adadf832e3
commit 94d15915b6
27 changed files with 2604 additions and 144 deletions
@@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("city", city)
q.put("status", 1)
// q.put("name","辽源一村")
def dbcursor = LOU_DONG_STAT.find(q)
def map = [:]
def set = new HashSet()
for(def db : dbcursor){
def list = map.get(db.region+"##"+db.name)
if(!list){
list = []
map.put(db.region+"##"+db.name, list)
}
list << db
if(db.house_num && db.house_num!="--"){
def key = db.region+"##"+db.name +"##"+db.house_num
if(!set.contains(key)){
set << key
list << db
}
}
}
return map
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){
def rets = []
def keys = []
distMap.each{k,dist->
def buildings = buildingMap.get(k)
if(buildings){
@@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{
def ret = []
def areaData = distCodeMap.get(obj.region)
if(areaData){
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << ""
ret << ""
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
def key = dist._id.toString() +"##"+obj.house_num
if(!keys.contains(key)){
keys << key
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << obj.house_num
ret << ""
}
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
if(arrs){
ret << arrs[0].trim()
}else{
ret << ''
}
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
ret << ""
}
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
if(arrs){
ret << arrs[0].trim()
}else{
ret << ''
}
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1){
// ret << "1"
// }else{
// ret << "0"
// }
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1) {
// ret << obj.elevators//电梯数
// }else{
// ret << ""
// }
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
}
}
}
}
@@ -367,7 +367,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","特房锦绣一方")
// q.put("name","辽源一村")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -25,8 +25,8 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()
task.delDealByPe()
task.deleteThirdPartyData()
// task.delDealByPe()
}
def delDealByPe(){
@@ -370,26 +370,26 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = ["东莞"] //,"杭州""广州",
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202110
def month = 202203
def rows = []
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202111)
def rets = findPeGteThirdParty(city, 202204)
def total = 0
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance.executeUpdate(sql)
sqlInstance236.executeUpdate(sql)
}
println city +" "+total
println city +" "+rets.size()
}
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>50"
def rets = []
sqlInstance2.eachRow(sql,{r->
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.commid = r.commid
map.price = r.target_avg_price
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -481,8 +481,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2022-02-01"
def endDate = "2022-03-01"
def startDate = "2022-03-01"
def endDate = "2022-04-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -2,25 +2,243 @@ package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
class CcbDistrictToDB {
static main(args) {
def task = new CcbDistrictToDB();
// task.run()
task.run1()
// task.run1()
// task.meituanRun()
// task.meituanSqrentRun()
// task.updateDistrict()
task.updateDongguangRegion()
}
def updateDongguangRegion(){
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"])
for(def d : dist){
def t = d.c_date
if(t.getTime()>cDate.getTime()){
}
}
}
def updateDistrict(){
def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"]
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
def index = 0
for(def city : citys){
def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"])
for(def d : dist){
def t = d.c_date
if(t.getTime()>cDate.getTime()){
if(d.name.contains("(")){
def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1)
def newName = d.name.replace(delstr,"")
index++
println index+" "+d.city+" "+d.name+" "+newName
def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district")
if(!obj){
println "update"
def udata = new BasicDBObject()
udata.put("name", newName)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
d.name = newName
addPrice(d)
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0)
}else{
println "delete"
DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
}
}
}
}
}
}
def meituanSqrentRun(){
ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion) {
def name = d['H']
name = nameFileter(name)
if(name){
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
}
}
},0);
}
def meituanRun(){
def index = 0//美团需入库小区合并后.xlsx
def keys = new CopyOnWriteArrayList()
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['H']
name = nameFileter(name)
def formatAddress = null
if(d['E']){
formatAddress = AddressUtil.formatAddress(city, d['E'])
}
if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
def map = findStandardObject2(city,region, name, d['E'], rets);
if(map && map.address){
formatAddress = AddressUtil.formatAddress(city, map.address)
if(!formatAddress){
if(map.address.contains("交叉口")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
}else if(map.address.contains("交叉")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
}
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
def stop = false
while (!stop){
if(data._id.toString()){
addPrice(data)
stop = true;
}
}
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
},0);
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def run1(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
def city = d['A']
def region = d['B']
def newRegion = RegionData.formatRegion(city, region)
@@ -71,6 +289,7 @@ class CcbDistrictToDB {
if(name){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
@@ -221,4 +440,77 @@ class CcbDistrictToDB {
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
return name;
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -16,7 +16,9 @@ import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
/**
* 重庆估值测试--2小时
*/
class EvaluationBatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
@@ -27,8 +29,8 @@ class EvaluationBatch {
static main(args) {
def task = new EvaluationBatch()
// task.run()
task.run1();
// task.run2()
// task.run1();
task.run2()
}
def run2(){
@@ -8,42 +8,17 @@ import com.fangjia.script.util.OpenApiClient
class BlankMatch {
static main(args) {
def task = new BlankMatch()
// task.run()
task.run()
// task.run1()
// task.run2()
task.run3()
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
if(dist){
row['C'] = dist._id.toString()
}
}
},0)
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
for(def d : datas){
if(d['B'] == row['B']){
row['C'] = d['C']
row['D'] = d['D']
row['E'] = d['E']
row['F'] = d['J']
break;
}
}
}
},0)
// task.run3()
}
def run() {
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
//def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx";
def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['C']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
@@ -54,8 +29,6 @@ class BlankMatch {
row['G'] = jsonObject.get("address")
}
}
});
}
@@ -81,4 +54,31 @@ class BlankMatch {
// }
});
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
if(dist){
row['C'] = dist._id.toString()
}
}
},0)
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
for(def d : datas){
if(d['B'] == row['B']){
row['C'] = d['C']
row['D'] = d['D']
row['E'] = d['E']
row['F'] = d['J']
break;
}
}
}
},0)
}
}
@@ -0,0 +1,239 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
class MatchEvaluateRegionTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new MatchEvaluateRegionTest()
task.run()//小区匹配
task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
task.addAuthor()
}
def run(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("name")
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def run1(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("name")
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def addPrice(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['L']){
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
if(priceObj){
row['M'] = priceObj.avg_price
row['N']= (int)(priceObj.avg_price * row['E'])
}
def avg_price = findAvgPrice(row['M'])
if(row['M'] && avg_price){
def weekendPrice = row['M'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def addAuthor(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def total = 747
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(index < avg){
row['P'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
});
}
def addDistrict(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def flag = d['M']
if(!flag){
def city = d['E']
def region = d['F']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['G']
name = nameFileter(name)
def formatAddress = null
def address = d['H']
if(address){
formatAddress = AddressUtil.formatAddress(city, address)
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
addPrice(data)
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
}
},0)
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name.contains("(")){
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
name = name.replace(delstr,"")
}
return name;
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -0,0 +1,280 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import com.fangjia.script.gj.district.FangjiaClientUtils
class MatchEvaluateTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx"
def total = 1500
static main(args) {
def task = new MatchEvaluateTest()
// task.run()//小区匹配
// task.run1()//地图匹配
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
// task.addPrice()
// task.addArroundPrice()
// task.addAuthor()
task.stat()
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A'] && !row['K']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['K'] = priceObj.unitPrice
row['L'] = (int) (priceObj.unitPrice * row['C'])
row['O'] = "周边价格"
}
}
});
}
def run(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("address")
row['I'] = jsonObject.get("propertyType")
row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def run1(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("address")
row['I'] = jsonObject.get("propertyType")
row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def addPrice(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['J'] && row['E']){
def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G'])
if(priceObj){
row['K'] = priceObj.avg_price
row['L']= (int)(priceObj.avg_price * row['C'])
}
def avg_price = findAvgPrice(row['D'])
if(row['K'] && avg_price){
def weekendPrice = row['K'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def addAuthor(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['N'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def addDistrict(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def flag = d['M']
if(!flag){
def city = d['E']
def region = d['F']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['G']
name = nameFileter(name)
def formatAddress = null
def address = d['H']
if(address){
formatAddress = AddressUtil.formatAddress(city, address)
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
addPrice(data)
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
}
},0)
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name.contains("(")){
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
name = name.replace(delstr,"")
}
return name;
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
def stat(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1)
ExcelUtils.multiThreadingModify(path, { d ->
if(d['B']){
for(def row : datas){
if(d['B']==row['B']){
if(row['K']){
d['P'] = row['K']
break
}
}
}
}
});
}
}
@@ -830,6 +830,7 @@ class CommonDatautils {
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
keys.put("sid", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -3166,5 +3167,22 @@ class CommonDatautils {
}
return avg_price;
}
static def findCityAvgRentRadio(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = SQMETRE_RENT_COLL.find(q)
def avg_price = null
def radios = []
for(def d : dbcursor){
if(d.price_to_rent){
radios << d.price_to_rent.toInteger()
}
}
def avg_radios = (int)(radios.sum()/radios.size())
return avg_radios;
}
}
@@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils;
class DataHandleUtils {
static main(args){
}
static def getStandardRegion(def regionMap, def region){
def newRegion = null;
@@ -94,6 +92,8 @@ class DataHandleUtils {
BigDecimal deSource = new BigDecimal(dSource);
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
}
static def trimChar(String c) {
try {
@@ -25,11 +25,11 @@ class DataMatch {
static main(args) {
def task = new DataMatch()
// task.run1()
task.run2()
// task.run2()
// task.run3()
// println task.findAddress("昌州街道棠香路")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
}
def run1(){
@@ -0,0 +1,246 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeDetailCrawl {
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
static main(args) {
def task = new AnjukeDetailCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://chongqing.anjuke.com/community/?from=navigation"
WebDriver driver = new ChromeDriver()
def regions = getRegions(driver, url)
getAllRegions(driver, regions)
}
def getAllRegions(driver, regions){
for(def url : regions){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def total = getTotal(doc)
if(total) {
if (total > 1250) {
} else {
def links = getAllLinks(url, driver)
parse(links,driver)
}
}
}
}
def getAllLinks(url, driver){
def stop = false
def urls = []
while(!stop) {
def doc = Jsoup.parse(driver.getPageSource());
def details = JQueryUtils.select(".list-cell a.li-row", doc)
for (def d : details) {
urls << d.attr("href")
}
def nextEle = JQueryUtils.select(".next", doc)
if (nextEle) {
driver.findElement(By.cssSelector(".next")).click()
}else{
stop = true
}
if(urls.size()>20){
stop = true
}
}
return urls
}
def getTotal(doc){
def totalText = JQueryUtils.selectText(".total-info", doc)
if(totalText){
def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim()
if(total && total.isNumber()){
return total.toInteger()
}
}
return null;
}
def getRegions(driver, url){
driver.get(url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
// if(loginIfm){
// def href = loginIfm[0].attr("src")
// driver.get(href)
// driver.findElementById("pwdTab").click()
// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框
// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框
// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。
// elemUsername.sendKeys("17740818975");
// elemPassword.sendKeys("12qwaszx");
// btn.submit(); // 点击行为;
//
// def regionEles = JQueryUtils.select(".region-parents li a", doc)
// def urls = []
// for(def e : regionEles){
// urls << e.attr("href")
// }
// return urls;
// }
def regionEles = JQueryUtils.select(".region-parents li a", doc)
def urls = []
for(def e : regionEles){
urls << e.attr("href")
}
return urls;
}
def parse(def links, driver){
def rets = []
for(def link : links){
def district = parsing(link, driver)
if(district){
rets << district
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
rets = []
}
}
}
if (rets.size()> 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
def parsing(url, driver){
def district = [:]
def stop = false
def count = 0
while(!stop){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def headers = JQueryUtils.select(".crumbs a", doc)
if(headers){
if (headers && headers.size() >= 3) {
district.city = headers[1].text().replace("小区", "")
district.region = headers[2].text().replace("小区", "")
district.name = headers[headers.size() - 1].text().replace("二手房", "")
}
district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc)
def priceStr = JQueryUtils.selectText(".average", doc)
if (priceStr && priceStr.isNumber()) {
district.avg_price = priceStr.toInteger()
}
def infoDiv = JQueryUtils.select(".info-list", doc)
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
if (houseCountStr) {
houseCountStr = houseCountStr.replace("户", "")
if (houseCountStr) {
district.house_count = houseCountStr.toInteger()
}
}
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
if(carSiteStr){
if(carSiteStr.contains("(")){
carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("("))
}
if(carSiteStr && carSiteStr.isNumber()){
district.car_site = carSiteStr
}
}
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.url = url
district.c_date = new Date()
district.m_date = new Date()
district.root = "anjuke.com"
stop = true;
}else{
Thread.sleep(3000)
count++
}
if(count>10){
stop = true;
}
}
return district
}
String getValue(String s){
if(s && s.contains("暂无")){
return null
}
return s
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.gj.crawl
import org.jsoup.Jsoup
class CommonCrawl {
static def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
}
@@ -17,7 +17,7 @@ class TempMatchDistrict {
}
def stat(){
def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"]
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
def total = 0
for(def city : citys){
def count = CommonDatautils.findAllDistrictCount("district", city)
@@ -83,6 +83,23 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100"
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static Map<String, Object> JsonToMap(JSONObject json) {
Map<String, Object> columnValMap = new HashMap<String, Object>();
@@ -0,0 +1,354 @@
package com.fangjia.script.gj.meituan
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.bson.types.ObjectId
class MatchTest {
static main(args) {
def task = new MatchTest()
// task.run()
// task.addNum()
task.dongguang()
// task.meituanDel()
}
def meituanDel(){
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
def index = 0
ExcelUtils.modify(path, { row ->
def flag = row['O']
if(flag){
def city = row['F']
def region = row['G']
def name = nameFileter(row['H'])
def obj = null
if(city == "东莞"){
obj = CommonDatautils.findDistrict(city, null, name, "district")
}else{
obj = CommonDatautils.findDistrict(city, region, name, "district")
}
if(obj){
index++
println index +" "+ city +' '+ region+" "+name
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(obj._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
},0);
}
def dongguang(){
def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx
ExcelUtils.modify(path, { row ->
def region = null
if(row['B'] && row['B']=="NULL" ){
region = null
}
def name = nameFileter(row['H'])
def address = row['E']
if (name && name.contains("湖畔新家")) {
def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district")
if(obj){
def json = GaoDeApiUtils.geo(row['F'], address)
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
def twinshop = getStreetInfo(m.location)
if(twinshop){
def newRegion = RegionData.formatRegion("东莞",twinshop)
if(newRegion){
def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district")
if(dbObj){
println "delete"
DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id))
}else{
println "update"
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
}
}
}
// println address +" "+twinshop
}
}
}
}
},0)
}
def getStreetInfo(def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
}
}
}
return township;
}
def addNum(){
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx
def index = 0
ExcelUtils.modify(path, { row ->
index++
row['A'] = index
},0)
}
def run() {
def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
def radioMap = [:]
for(def city : citys){
// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
// radioMap.put(city, avg_radio)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
ExcelUtils.multiThreadingModify(path, { row ->
def region = null
if(row['C'] && row['C']=="NULL" ){
region = null
}
def name = row['D']
def address = row['F']
def city = RegionData.formatCity(row['B'])
if(name && !row['N']){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
if(rMap.level){
row['P'] = rMap.level
}
}
}
}
if (name && row['N']) {
//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name)
// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null)
// if (jsonObject != null && jsonObject.precise != -1) {
// row['F'] = jsonObject.get("city")
// row['G'] = jsonObject.get("district")
// row['H'] = jsonObject.get("name")
// row['I'] = jsonObject.get("propertyType")
// row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
// }
if(row['G'] && row['H'] && row['I']){
// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H'])
// if(priceObj){
// row['M'] = priceObj.avg_price
// }
// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H'])
// if(sqmetreRentPrice){
// row['N'] = sqmetreRentPrice
// }
if(row['N'] && !row['O']){
def avg_rent_radio = radioMap.get(city)
row['O'] = (int)(row['N']/avg_rent_radio)
}
}
}
})
}
def getArroundDistrictPriceByAddress(def city, def address, def levels){
def json = GaoDeApiUtils.geo(city, address)
def location = null
def detail = new BasicDBObject()
def level = null
def rank = null
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
location = m.location
level = m.level
}
}
if(location){
def arrs = location.split(",")
detail.lng = arrs[0].toString().toDouble()*10000000000L
detail.lat = arrs[1].toString().toDouble()*10000000000L
}
if(detail.lng){
detail.city = city
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
if(list && list.size()>10){
list = sortByDistance(detail, list);
list = list[0..9]
}
def arroundAvgPrice = getArroundAvgPrice(list)
if(arroundAvgPrice){
def mod = Integer.MAX_VALUE;
for(def d : levels){
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
if(newMod<mod){
mod = newMod
rank = d.level
}
}
}
}
def m = [:]
m.level = level
m.rank = rank
return m
}
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
}
//排序
Collections.sort(list, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return list;
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
return datas;
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
return null
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
return name;
}
}
@@ -42,7 +42,6 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def proviceMap = getDocument(driver);
def citys = getCity(driver,proviceMap)
@@ -59,9 +58,11 @@ def getDetailLinks(driver, cityList){
citys.each{city,id->
def urls = []
if(!zxcitys.contains(cityMap.provice)){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
getDocument(url, driver)
}else{
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
getDocument(url, driver)
}
def doc = Jsoup.parse(driver.getPageSource());
@@ -89,7 +90,7 @@ def getDetailLinks(driver, cityList){
}
}
def rets = batchParse(urls);
def rets = batchParse(urls, driver);
saveToDB(rets);
println city +" "+urls.size()
}
@@ -124,7 +125,7 @@ def getCity(driver, proviceMap){
for(def c : cityEles){
def cityid = c.attr("data-cityid")
def city = c.text()
if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
cityMap.put(city, cityid)
}
@@ -149,7 +150,7 @@ def getCity(driver, proviceMap){
def getDocument(driver){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport=");
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
def doc = Jsoup.parse(driver.getPageSource());
@@ -159,7 +160,7 @@ def getDocument(driver){
for(def ele : proviceEles){
def provice = ele.text()
def value = ele.attr("data-proviceid")
if(value && provice!="北京市" && provice!="天津市"){
if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){
provinceMap.put(provice, value)
}
}
@@ -167,8 +168,8 @@ def getDocument(driver){
return provinceMap;
}
def batchParse(def urls){
WebDriver driver = new ChromeDriver()
def batchParse(def urls, driver){
def ret = []
for(def map :urls){
def doc = getDocument(map.url, driver)
@@ -280,7 +281,7 @@ def parse(province, city, url,doc){
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc)
if(startDateStr){
if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
}
@@ -329,6 +330,13 @@ def parse(province, city, url,doc){
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
if(result.startDate){
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
}
result.content = DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
result.url = url
@@ -16,6 +16,27 @@ import java.util.regex.Matcher
import java.util.regex.Pattern
class DataUtil {
static def checkMinDate(curDate, checkDate){
if(curDate){
def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
if(curDate.getTime()<minDate.getTime()){
return null
}
}
return curDate
}
static def checkMaxDate(curDate, checkDate){
if(curDate){
def maxDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
if(curDate.getTime()>maxDate.getTime()){
return null
}
}
return curDate
}
static def getAreaNumber(def input, def maxArea) {
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)')
Matcher matcher = pattern.matcher(input)
@@ -112,10 +133,10 @@ class DataUtil {
if (list.size() >= 1) {
String s1 = list.get(0)
if (s1.contains("万元")) {
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "")
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","")
return (int) (s1.toDouble() * 10000)
} else {
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "")
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","")
if (s1.isInteger()) {
return s1.toInteger()
} else {
@@ -0,0 +1,348 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import javafx.scene.input.DataFormat
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class IcbcCrawl {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new IcbcCrawl();
task.run();
}
def run(){
def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver.get(url)
driver.findElementById("category_SM310000000000000006").click()
Thread.sleep(3000)
def urls = getAllDetailLinks(driver)
println "共:"+urls.size()
detailCrawling(urls, driver)
}
def getAllDetailLinks(driver){
def stop = false
def urls = []
while (!stop){
def doc = Jsoup.parse(driver.getPageSource());
def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc)
if(eles){
for(def e : eles){
def href = e.select(".li_2_name a").attr("href")
urls << "https://gf.trade.icbc.com.cn"+href
}
def nextPage = JQueryUtils.select("a:contains(下一页)", doc)
if(nextPage){
driver.findElement(By.linkText("下一页")).click();
}else{
stop = true
}
}else{
stop = true
}
// if(urls.size()>=10){
// stop = true
// }
}
return urls;
}
def detailCrawling(def urls, driver){
def rets = []
for(def url : urls){
driver.get(url)
Thread.sleep(5000)
def doc = Jsoup.parse(driver.getPageSource());
def result = parser(doc, url)
if (result) {
rets << result
println rets.size()
}
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
rets = []
}
}
if(rets.size()>0){
saveData(rets)
print "完"
}
}
def parser(doc, url){
def result = [:]
if(doc){
def title = JQueryUtils.selectText("#prodnameb", doc)
def id = url.substring(url.indexOf("="), url.length())
if(title){
def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc)
if(cityStr && cityStr.contains("...")){
def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc)
if(cityEles){
cityStr = cityEles[0].attr("title")
}
}
if(cityStr && cityStr.contains("省")){
result.province = cityStr.substring(0, cityStr.indexOf("省"))
result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length())
}else{
if(cityStr.contains("自治区")){
result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3)
result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length())
}else{
result.city = cityStr.substring(0, cityStr.indexOf("市"))
}
}
if(!result.city){
return null
}
result.title = title
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","")
def t1 = JQueryUtils.selectText("#pricep", doc)
if(t1 && t1.contains("原因")){
result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length())
}
result.type = JQueryUtils.selectText("dl:contains(类  型:) dd", doc)
def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc)
if(enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def lookCount = JQueryUtils.selectText(["#visitorCount"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
}
def callCount = JQueryUtils.selectText(["#remindPPNum"], doc)
if(callCount) {
result.callCount = callCount.toInteger()
}
def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim()
if(curPriceStr.isInteger()) {
result.curPrice = curPriceStr.toInteger()
}else {
result.curPrice = (int)(curPriceStr.toDouble())
}
}
def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc)
if(startPriceEles) {
def startPriceStr = startPriceEles[0].attr("title")
if(startPriceStr){
startPriceStr = startPriceStr.replace(",", "").replace("¥","")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}else {
result.startPrice = (int)(startPriceStr.toDouble())
}
}
}
def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
if(addPriceEles){
def addPriceStr = addPriceEles[0].attr("title")
if(addPriceStr){
addPriceStr = addPriceStr.replace(",", "").replace("¥","")
if(addPriceStr) {
result.addPrice = addPriceStr.toDouble().toInteger()
}
}
}
def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc)
if(evaluationPriceEles){
def evaluationPriceStr = evaluationPriceEles[0].attr("title")
if(evaluationPriceStr){
evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","")
if(evaluationPriceStr.isInteger()) {
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
}else {
result.evaluationPrice = (int)(evaluationPriceStr.toDouble())
}
}
}
def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
if(depositPriceEles){
def v = depositPriceEles[0].attr("title")
if(v){
v = v.replace(",", "").replace("¥","")
if(v.isInteger()) {
result.depositPrice = v.toDouble().toInteger()
}else {
result.depositPrice = (int)(v.toDouble())
}
}
}
result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc)
result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc)
result.court = JQueryUtils.selectText("#goToCourt", doc)
def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc)
if(mobileEles){
result.mobile = mobileEles[0].attr("title")
}
def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc)
if(s1) {
def start = s1.indexOf("法院将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
}
def dateRangs = PaimaiDataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
}
}
def trs = JQueryUtils.select([".rygtable table tr"], doc)
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
if(d_tds[2].text() && d_tds[2].text()!="/"){
detailMap.put(d_tds[1].text(), d_tds[2].text())
}
}else {
if(d_tds[1].text() && d_tds[1].text()!="/") {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}
}
}
if(detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if(!result.address) {
result.address = detailMap.get("拍品名称")
if(!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs =JQueryUtils.select("#scroll li img", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("src")
if(href.contains("icbc.com")) {
imgLinks << "https:"+href
}
}
result.imgUrls = imgLinks
}
def detailStr = JQueryUtils.selectText(".detail_list", doc)
if(detailStr){
def area = DataUtil.getAreaNumber(detailStr, 5000)
if(area) {
result.area = area.toDouble()
if(result.tradePrice) {
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
}
}
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = detailStr.toString()
result.url = url
result.mDate = new Date()
result.source = "融e够"
return result;
}
}
return null
}
def getCallCount(id){
def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id
def s = ABuYunUtil.doPost(url, null, null, "utf-8", true)
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
println jsonObject
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
}
@@ -39,8 +39,6 @@ class JdPaimaiCrawl {
println "共:"+ids.size()
parserList( existsIds,ids, driver)
}
def getList(driver){
@@ -478,6 +476,13 @@ class JdPaimaiCrawl {
}
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
if(result.startDate){
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
result.url = url
@@ -1,7 +1,12 @@
package com.fangjia.script.gj.paimai
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.DateUtils
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class TaobaoPaimaiCrawl {
static main(args) {
@@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl {
task.run();
}
def run(){
def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1"
def regionUrls = regionCrawl(seedUrl);
def pageUrls = pageUrlCrawl(regionUrls)
detailCrawl(pageUrls)
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1"
def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1"
WebDriver driver = new ChromeDriver()
getAllCityRegions(driver, url);
}
def regionCrawl(def seedUrl){
def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk");
def regionUrls = []
if(document){
def eles = JQueryUtils.select(".small-subcondion a", document)
if(eles){
for(def el : eles){
def reiongUrl = el.attr("href").trim()
if(reiongUrl.startsWith("//")){
regionUrls << "http:"+reiongUrl
def getAllCityRegions(driver, url){
def provinces = getAllProvince(driver, url)
def citys = ["天津","上海","北京","重庆"]
for(def province : provinces){
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
def doc = Jsoup.parse(driver.getPageSource());
def cityEles= JQueryUtils.select(".unfold .J_FilterCity a")
for(def cityEle : cityEles){
def city = cityEle.text()
if(citys.contains(city)){
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
doc = Jsoup.parse(driver.getPageSource());
driver.findElement(By.cssSelector(".J_FilterCity a")).click()
doc = Jsoup.parse(driver.getPageSource());
def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a")
for(def r : regionEles){
def m = [:]
m.province = province
m.city = city
m.region = r.text()
def urls = crawlLinks(m, driver)
parse(m, driver, urls)
}
}else{
def m = [:]
m.province = province
m.city = city
def urls = crawlLinks(m, driver)
parse(m, driver, urls)
}
}
}
return regionUrls;
}
def pageUrlCrawl(regionUrls){
def crawlLinks(def m, def driver){
def stop = false
def urls = []
for(def url : regionUrls){
def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk");
if(document){
def pages = JQueryUtils.selectText(".page-total",document)
if(pages){
pages = pages.toInteger()
for(def page=1;page<pages;page++){
urls << url+"&page="+page
while(!stop){
def doc = Jsoup.parse(driver.getPageSource());
def details = JQueryUtils.select(".sf-pai-item-list li a", doc)
for(def d : details){
urls << "https:"+d.attr("href")
}
def nextEle = JQueryUtils.select(".next", doc)
if(nextEle){
driver.findElement(By.cssSelector(".next")).click()
}
}
return urls
}
def getAllProvince(driver, url){
driver.get(url);
Thread.sleep(100000)
def doc = Jsoup.parse(driver.getPageSource());
def eles= JQueryUtils.select("li:contains(标的物类型) .condition li a", doc)
def provinces = []
for(def e : eles){
def province = e.text()
provinces << province
}
return provinces
}
def parse(m, driver, urls){
def results = []
for(def url : urls) {
driver.get(url);
def doc = Jsoup.parse(driver.getPageSource());
def result = [:]
if (doc) {
def title = JQueryUtils.selectText([".pm-main h1"], doc)
def statusDesc = JQueryUtils.selectText([".bid-fail"], doc)
result.province = m.province
result.city = m.city
if (m.region) {
result.region = m.region
}
if (title) {
def addr = title.toString().replace("本场拍卖已", "").replace(" !", "")
if (addr && addr.indexOf("室") > -1) {
result.address = addr.substring(0, addr.indexOf("室") + 1)
} else {
result.address = addr
}
if (result.city) {
if (result.address.contains(city + "市")) {
result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1)
} else {
result.address = result.address.substring(result.address.indexOf(city) + city.length())
}
}
if (result.address.indexOf("】") > -1) {
result.address = result.address.substring(result.address.indexOf("】") + 1)
}
if (result.city) {
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
}
if (statusDesc) {
def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1)
result.status = DataUtil.getPaiMaiType(status)
result.statusDesc = statusDesc
}
def statusEle = JQueryUtils.select(".status-tip", doc)
if (statusEle) {
def sEle = JQueryUtils.select(".status-ing", doc)
if (sEle) {
result.status = "进行中"
} else {
result.status = "即将开始"
}
}
def curPrice = JQueryUtils.selectText(["#sf-price"], doc)
if (curPrice) {
if (curPrice.contains("当前价")) {
curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (curPrice) {
curPrice = curPrice.replace(",", "")
if (curPrice.isInteger()) {
result.curPrice = curPrice.toInteger()
} else {
result.curPrice = (int) (curPrice.toDouble())
}
}
} else if (curPrice.contains("成交价")) {
def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (tradePrice) {
tradePrice = tradePrice.replace(",", "")
if (tradePrice.isInteger()) {
result.tradePrice = tradePrice.toInteger()
} else {
result.tradePrice = (int) (tradePrice.toDouble())
}
}
} else if (curPrice.contains("起拍价")) {
def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (startPrice) {
startPrice = startPrice.replace(",", "")
if (startPrice.isInteger()) {
result.startPrice = startPrice.toInteger()
} else {
result.startPrice = (int) (startPrice.toDouble())
}
}
}
}
def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc)
if (finishStr) {
if (finishStr.contains("结束时间")) {
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
if (finishStr) {
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
}
} else if (finishStr.contains("成交时间")) {
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
if (finishStr) {
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
}
}
}
def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc)
if (enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc)
if (callCount) {
result.callCount = callCount.toInteger()
}
def lookCount = JQueryUtils.selectText(["#J_Looker"], doc)
if (lookCount) {
result.lookCount = lookCount.toInteger()
}
def tds = JQueryUtils.select(["#J_HoverShow td"], doc)
if (tds && tds.size() > 0) {
for (def td : tds) {
def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "")
if (str) {
if (str.contains("变卖价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.sellOffPrice = p.toInteger()
} else {
result.sellOffPrice = (int) (p.toDouble())
}
}
} else if (str.contains("加价幅度")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.addPrice = p.toInteger()
} else {
result.addPrice = (int) (p.toDouble())
}
}
} else if (str.contains("类型")) {
result.type = td.select(".pay-type").text()
} else if (str.contains("保证金")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.depositPrice = p.toInteger()
} else {
result.depositPrice = (int) (p.toDouble())
}
}
} else if (str.contains("变卖周期")) {
result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("竞价周期")) {
result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("优先购买权")) {
result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("变卖预缴款")) {
def str1 = td.select("span:eq(1) .J_Price").text()
def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.sellOffAdvancePay = p.toInteger()
} else {
result.sellOffAdvancePay = (int) (p.toDouble())
}
}
} else if (str.contains("评估价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.evaluationPrice = p.toInteger()
} else {
result.evaluationPrice = (int) (p.toDouble())
}
}
} else if (str.contains("延时周期")) {
result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("起拍价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.startPrice = p.toInteger()
} else {
result.startPrice = (int) (p.toDouble())
}
}
}
}
}
}
result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc)
def trs = JQueryUtils.select(["#J_desc table tr"], doc)
if (trs) {
def detailMap = [:]
for (def tr : trs) {
def d_tds = tr.select("td")
if (d_tds && d_tds.size() >= 2) {
if (d_tds.size() == 3) {
detailMap.put(d_tds[1].text(), d_tds[2].text())
} else {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
} else {
if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
}
}
}
if (detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if (!result.address) {
result.address = detailMap.get("拍品名称")
if (!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs = JQueryUtils.select([".slide-bigpic img"], doc)
if (imgs) {
def imgLinks = []
for (def img : imgs) {
def href = img.attr("src")
if (href.contains("http")) {
imgLinks << href
} else {
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
}
}
result.imgUrls = imgLinks
}
}
//面积
def article = JQueryUtils.selectText(["#J_ItemNotice"], doc)
if (article) {
def area = DataUtil.getAreaNumber(article, 5000)
if (!area) {
article = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
if (article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if (!area) {
article = JQueryUtils.selectText(["body"], doc)
if (article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if (area) {
result.area = area.toDouble()
if (result.tradePrice) {
result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble())
}
}
}
}
return urls;
}
def detailCrawl(pageUrls){
if(pageUrls){
for(def pageUrl : pageUrls){
def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk");
if(document){
def pages = JQueryUtils.select(".page-total", document)
def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
if (s1) {
def dateRangs = DataUtil.getDateRange(s1)
if (dateRangs) {
for (def j = 0; j < dateRangs.size(); j++) {
def date = dateRangs[j]
def pattern = null
if (date.endsWith("日时") || date.endsWith("日")) {
pattern = "yyyy年MM月dd日"
} else {
def ind = date.indexOf("日")
if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) {
date = date + "时"
}
pattern = "yyyy年MM月dd日HH时"
}
if (j == 0) {
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
} else {
if (!result.finishDate) {
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
}
}
}
}
if (!result.startPrice) {
result.startPrice = DataUtil.getStartPrice(s1)
}
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
def estateType = DataUtil.findEstateType(s)
if (estateType) {
result.matchEstateType = estateType
}
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
result.url = url
result.mDate = new Date()
results << result
}
return results;
}
@@ -1,20 +1,38 @@
package com.fangjia.script.loudong
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class CqfdcMatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new CqfdcMatch();
// task.run()
// task.stat()
task.export2()
// task.export2()
// task.matchLoudong()
task.addLoudong()
}
def export(){
def q = new BasicDBObject()
@@ -139,6 +157,29 @@ class CqfdcMatch {
return rets;
}
def findAllLoundongByHouseNum(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000)
def rets = []
def map = [:]
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num
if(d.unit_num){
key = key +"##"+d.unit_num
}
def detailMap= map.get(key)
if(!detailMap){
detailMap = [:]
map.put(key,detailMap)
}
if(detailMap.height){
}
}
return rets;
}
def stat(){
def q = new BasicDBObject()
@@ -168,6 +209,91 @@ class CqfdcMatch {
}
def matchLoudong(){
def names = findAllLoudong("重庆")
print names.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(names.size())
def retMap = new ConcurrentHashMap()
for(def i=0;i<names.size();i++){
def key = names[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def arrs = key.split("#")
def jsonObject = OpenApiClient.addressMatch(arrs[0], arrs[2], "property", false, arrs[1], null)
if(jsonObject != null && jsonObject.precise!=-1) {
def m = [:]
m.id = jsonObject.get("id")
m.city = jsonObject.get("city")
m.region = jsonObject.get("district")
m.name = jsonObject.get("name")
def dist = CommonDatautils.findDistrictById(m.id)
if(dist){
m.sid = dist.sid
}
retMap.put(key, m)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
println "匹配后:"+retMap.size()
updateLoudong(retMap)
println "修改完成!"
}
def updateLoudong(retMap){
def index = 0
retMap.each{key,m->
def arrs = key.split("#")
def q = new BasicDBObject("city", arrs[0])
q.put('region', arrs[1]);
q.put('name', arrs[2]);
def udata = new BasicDBObject()
udata.put("district_id", m.id.toString())
udata.put("name_in_seaweed", m.name)
udata.put("district_no", m.sid)
HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true);
index++
println index
}
}
def findAllLoudong(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("district_id",new BasicDBObject('$exists', false))
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("name", 1)
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
def names = new CopyOnWriteArrayList()
for(def db : dbcursor){
def key = db.city +"#"+db.region+"#"+db.name
if(!names.contains(key)){
names << key
}
}
return names;
}
def addLoudong(){
}
}
@@ -192,7 +192,7 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
def citys = ["德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
for(def city : citys){
def str = cityList.get(city)
for(def i=0;i<1;i++){
@@ -245,6 +245,48 @@ public class CommonUtils {
}
static void recalculationSqRent(String city, String region, String name, String estateType, Date calcDate = new Date(), boolean isTest = false, int calcType = 0, needRedrawTrend = true, def sync = false) {
calcDate = com.fangjia.core.util.DateUtil.truncateDay(calcDate)
if (estateType && estateType.contains("±ðÊû")) {
estateType = "±ðÊû"
} else if (estateType) {
estateType = "Ð¡Çø"
}
DBObject cmd = new BasicDBObject()
cmd.put("city", city)
cmd.put("region", region)
cmd.put("name", name)
cmd.put('estate_type', estateType)
cmd.put("calc_date", calcDate)
cmd.put("calc_type", 0)
println(cmd)
DBObject data = new BasicDBObject()
data.put("category", "square_rent")
data.put("priority", 3)
data.put("status", 0)
data.put("remark", 'week')
data.put("c_date", new Date())
if(isTest){
data.put("ip", "192.168.0.231")
COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false)
}else{
data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)])
COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false)
}
if(sync){
while (COMMAND_COLL.count(cmd) != 0){
Thread.sleep(1000)
}
}
}
/**
* 重算小区均价
* @param city
@@ -314,6 +314,43 @@ public class DbUtils{
}
return districts
}
/**
* 查询附近一共里内小区的数量
*/
def static findAroundDistrictByRadius(BasicDBObject district, def radius){
def districts = []
if(hasGeo(district)){
def query = new BasicDBObject()
query.append("cat", "district")
query.append("city", district.city)
query.append("visible", 0)
DBObject sdq = new BasicDBObject();
def lat = district.lat / 10000000000.0
def lng = district.lng / 10000000000.0
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) radius / 1000.0));
query.append('$sd', sdq)
def keys = new BasicDBObject()
keys.append("city", 1)
keys.append("region", 1)
keys.append("name", 1)
keys.append("lat", 1)
keys.append("lng", 1)
keys.append("lat2", 1)
keys.append("lng2", 1)
keys.append("estate_type2", 1)
DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo->
if(!district._id || dbo._id != district._id){
districts << dbo
}
}
}
return districts
}
/**
* 查询小区均价
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220401", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220501", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0