Merge branch 'script_gj' into 'master'
Script gj See merge request !297
This commit is contained in:
+20
-5
@@ -5,6 +5,7 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
@@ -22,16 +23,19 @@ class MatchEvaluateRegionTest {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
def total = 1500
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateRegionTest()
|
||||
task.run()//小区匹配
|
||||
task.run1()//地图匹配
|
||||
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
task.addPrice()
|
||||
task.addArroundPrice()
|
||||
task.addAuthor()
|
||||
}
|
||||
def run(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
|
||||
@@ -51,7 +55,6 @@ class MatchEvaluateRegionTest {
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
|
||||
@@ -71,7 +74,6 @@ class MatchEvaluateRegionTest {
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['L']){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
|
||||
@@ -91,9 +93,22 @@ class MatchEvaluateRegionTest {
|
||||
|
||||
}
|
||||
|
||||
def addArroundPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['A'] && !row['K']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['M'] = priceObj.unitPrice
|
||||
row['N'] = (int) (priceObj.unitPrice * row['C'])
|
||||
row['Q'] = "周边价格"
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addAuthor(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def total = 747
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
+1
-1
@@ -117,7 +117,7 @@ class MatchEvaluateTest {
|
||||
|
||||
def addAuthor(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
|
||||
@@ -0,0 +1,510 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.util.SourceNameUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class PocTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new PocTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
// match("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
|
||||
|
||||
// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
|
||||
// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
|
||||
// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
}
|
||||
|
||||
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['P'] && row['C']){
|
||||
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['P'] = jsonObject.get("name")
|
||||
row['Q'] = jsonObject.get("id")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['Q']){
|
||||
def dist = CommonDatautils.findDistrictById(row['Q'])
|
||||
if(dist){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['E'] = priceObj.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def standard(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['C']){
|
||||
def jsonObject = OpenApiClient.standard2Action(row['A'], row['C'])
|
||||
if(jsonObject){
|
||||
println jsonObject
|
||||
row['F'] = jsonObject.get("province")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("township")
|
||||
row['J'] = jsonObject.get("street")
|
||||
row['K'] = jsonObject.get("streetNumber")
|
||||
row['L'] = jsonObject.get("buildingNumber")
|
||||
row['M'] = jsonObject.get("unitNumber")
|
||||
row['N'] = jsonObject.get("floor")
|
||||
row['O'] = jsonObject.get("roomNumber")
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def addLoudongNum(path){
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if (row['Q'] && row['L']) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
|
||||
if (jsonObject && jsonObject.result) {
|
||||
if(jsonObject.result.size()>=1){
|
||||
def obj = jsonObject.result[0]
|
||||
row['S'] = obj.id
|
||||
row['R'] = obj.houseNumber
|
||||
}
|
||||
}
|
||||
|
||||
if(row['O']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
|
||||
if (jsonObject1 && jsonObject1.result) {
|
||||
if(jsonObject.result.size()>=1){
|
||||
def obj = jsonObject.result[0]
|
||||
row['U'] = obj.id
|
||||
row['T'] = obj.roomNum
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def createLoudong(path, output){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
|
||||
for(def row : datas){
|
||||
def data = []
|
||||
data << row['A']
|
||||
data << row['B']
|
||||
data << row['C']
|
||||
data << row['D']
|
||||
if (row['Q'] && row['L']) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
|
||||
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
|
||||
def obj = jsonObject.result[0]
|
||||
data << obj.houseNumber
|
||||
data << obj.id
|
||||
data << obj.type// 物业类型
|
||||
data << ""//主朝向
|
||||
data << obj.floors//地上层数
|
||||
data << ""//地下层数
|
||||
data << obj.buildYear;//竣工日期
|
||||
data << ""//梯
|
||||
data << ""//户
|
||||
data << obj.households//总户数
|
||||
data << ""//景观
|
||||
}else{
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""// 物业类型
|
||||
data << ""//主朝向
|
||||
data << ""//地上层数
|
||||
data << ""//地下层数
|
||||
data << ""//竣工日期
|
||||
data << ""//梯
|
||||
data << ""//户
|
||||
data << ""//总户数
|
||||
data << ""//景观
|
||||
}
|
||||
|
||||
if(row['O']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
|
||||
if (jsonObject1 && jsonObject1.result) {
|
||||
if(jsonObject1.result.size()>=1){
|
||||
def obj = jsonObject1.result[0]
|
||||
|
||||
data << obj.roomNum//户号
|
||||
data << obj.id//户ID
|
||||
data << obj.direction//主
|
||||
data << obj.sight//景观
|
||||
if(obj.roomNum && obj.roomNum.contains("跃")){
|
||||
data << "跃式"//户型结构
|
||||
}else{
|
||||
data << ""//户型结构
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write(output, rets, "底层数据-楼栋-户")
|
||||
}
|
||||
|
||||
def addDetail(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
|
||||
for(def d : datas){
|
||||
if(d['Q']){
|
||||
def dist = CommonDatautils.findDistrictById(d['Q'])
|
||||
if(dist){
|
||||
def row = []
|
||||
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
|
||||
if(jsonObject) {
|
||||
|
||||
row << d['A']
|
||||
row << d['B']
|
||||
row << d['C']
|
||||
row << jsonObject.get("name")//小区名称
|
||||
row << d['Q']//小区ID
|
||||
if(jsonObject.get("alias")){
|
||||
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << jsonObject.get("address")//小区地址
|
||||
row << jsonObject.get("commercialArea")//板块
|
||||
row << jsonObject.get("township")//街道***********
|
||||
row << jsonObject.get("avgPrice")//基准价
|
||||
row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度
|
||||
row << jsonObject.get("finishBuildTime")//竣工时间
|
||||
row << jsonObject.get("developer")//开发商名称
|
||||
row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
|
||||
row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
|
||||
row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
|
||||
row << jsonObject.get("propertyManagement")//物业公司
|
||||
row << jsonObject.get("managementFee")//物业费
|
||||
row << jsonObject.get("buildingCount")//住宅幢数
|
||||
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
|
||||
row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积
|
||||
|
||||
}
|
||||
|
||||
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
|
||||
if(facilityObj && facilityObj.result){
|
||||
def result = facilityObj.result
|
||||
if(result.keySchool){
|
||||
row << '是'
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.school){
|
||||
row << result.school.join(",").replace('"',"")
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
def bus = []
|
||||
if(result.bus){
|
||||
bus.addAll(result.bus)
|
||||
}
|
||||
if(result.metro){
|
||||
bus.addAll(result.metro)
|
||||
}
|
||||
|
||||
if(bus){
|
||||
row << bus.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.supermarket){
|
||||
row << result.supermarket.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.hospital){
|
||||
row << result.hospital.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.bank){
|
||||
row << result.bank.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
rows << row
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "底层数据-小区")
|
||||
}
|
||||
|
||||
def createPaimai(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
if(d['Q']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['Q'])
|
||||
if (dist) {
|
||||
def key = dist.city+"#"+dist.region+"#"+dist.name
|
||||
if(keys.contains(key)){
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10)
|
||||
if(jsonObject && jsonObject.result){
|
||||
def results = jsonObject.result
|
||||
for(def r: results){
|
||||
def row = []
|
||||
row << d['A']
|
||||
row << d['B']
|
||||
row << d['C']
|
||||
row << r.matchName
|
||||
row << r.address
|
||||
row << r.houseNum
|
||||
row << r.roomNum
|
||||
row << r.area
|
||||
row << r.startDate
|
||||
row << r.count
|
||||
row << r.status
|
||||
row << r.tradeAvgPrice
|
||||
row << r.tradePrice
|
||||
row << r.id
|
||||
rows << row
|
||||
}
|
||||
println dist.name +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "拍卖案例")
|
||||
}
|
||||
|
||||
def createHouse(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "挂牌"
|
||||
row << r.districtName
|
||||
if(r.estateType){
|
||||
row << r.estateType
|
||||
}else{
|
||||
if(r.purpose){
|
||||
row << r.purpose
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
|
||||
if(r.room){
|
||||
def roomStr = r.room+"室"
|
||||
if(r.hall){
|
||||
roomStr = roomStr +r.hall+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.area
|
||||
row << r.fitment
|
||||
row << r.pDate
|
||||
row << ""
|
||||
row << r.avgPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.height){
|
||||
row << r.height
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << r.direction
|
||||
row << SourceNameUtil.getResourceName(r.root)
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "挂牌案例")
|
||||
}
|
||||
|
||||
def createEvaluation(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "评估"
|
||||
row << r.commName
|
||||
row << ""
|
||||
|
||||
if(r.roomcnt){
|
||||
def roomStr = r.roomcnt+"室"
|
||||
if(r.livingroomcnt){
|
||||
roomStr = roomStr +r.livingroomcnt+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.buildArea
|
||||
row << r.fitment
|
||||
row << r.cDate.substring(0, r.cDate.indexOf("T"))
|
||||
row << ""
|
||||
row << r.unitPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.floorcnt){
|
||||
row << r.floorcnt
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.houseaspect && !r.houseaspect.contains("暂")){
|
||||
row << r.houseaspect
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << "房价网"
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "评估案例")
|
||||
}
|
||||
|
||||
|
||||
def createDealPrice(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "成交"
|
||||
row << r.districtName
|
||||
if(r.type){
|
||||
row << r.type
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.room){
|
||||
def roomStr = r.room+"室"
|
||||
if(r.hall){
|
||||
roomStr = roomStr +r.hall+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.area
|
||||
row << r.fitment
|
||||
row << r.tradeDate.substring(0, r.tradeDate.indexOf("T"))
|
||||
row << ""
|
||||
row << r.avgPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.height){
|
||||
row << r.height
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.direction && !r.direction.contains("暂")){
|
||||
row << r.direction
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << r.source
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "成交案例")
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch {
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def cities = ["北京", "上海"]
|
||||
def cities = ["北京", "上海"]//
|
||||
for (def city : cities) {
|
||||
def ret = CommonDatautils.getAllDistricts("district", city)
|
||||
// def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district");
|
||||
|
||||
+86
-1
@@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
@@ -19,9 +22,91 @@ class PaimaiTest {
|
||||
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new PaimaiTest();
|
||||
task.run();
|
||||
// task.run();
|
||||
// task.stat()
|
||||
// task.updatePaimai()
|
||||
task.updateFieldPaimai()
|
||||
}
|
||||
|
||||
def updateFieldPaimai(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
|
||||
def index = 0
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
for(def obj : dbcursor) {
|
||||
index++
|
||||
println index
|
||||
if(obj.depositPrice){
|
||||
obj.put("depositPrice", obj.depositPrice.toInteger())
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updatePaimai(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def index =0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
def query = new BasicDBObject()
|
||||
query.put("url", d.url)
|
||||
def dbo = paimai43_COLL.findOne(query);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
d.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimai43_COLL.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", new BasicDBObject('$in', ["重庆","重庆市"]))
|
||||
def dbcursor = paimaiResultColl.find(q)
|
||||
def keys = []
|
||||
for(def db : dbcursor){
|
||||
if(db.matchCity){
|
||||
def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName;
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
// try{
|
||||
// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null)
|
||||
// if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// db.matchCity = jsonObject.get("city")
|
||||
// db.matchRegion = jsonObject.get("district")
|
||||
// db.matchName = jsonObject.get("name")
|
||||
// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db))
|
||||
// println db.address+" "+db.matchName
|
||||
// }
|
||||
// }catch(Exception e){
|
||||
// println "error "+db.address
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
println keys.size()
|
||||
}
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
|
||||
+10
-2
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -16,8 +17,15 @@ class CqfdcChromeCrawl {
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new CqfdcChromeCrawl();
|
||||
task.run()
|
||||
// def task = new CqfdcChromeCrawl();
|
||||
// task.run()
|
||||
|
||||
def name = "E3号楼" //69幢(规划C-7号楼) 规划5-5号楼
|
||||
def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
println s
|
||||
|
||||
def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+单元")
|
||||
println unitNumStr
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+418
-14
@@ -1,10 +1,14 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
@@ -14,6 +18,8 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CqfdcMatch {
|
||||
|
||||
@@ -23,6 +29,10 @@ class CqfdcMatch {
|
||||
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
|
||||
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
|
||||
|
||||
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
|
||||
public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
@@ -33,6 +43,7 @@ class CqfdcMatch {
|
||||
// task.matchLoudong()
|
||||
|
||||
task.addLoudong()
|
||||
// task.addFdcLoudong()
|
||||
}
|
||||
def export(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -157,29 +168,378 @@ class CqfdcMatch {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addHouseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def keys = []
|
||||
def index = 0
|
||||
for(def d : dbcursor) {
|
||||
index++
|
||||
println index
|
||||
if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){
|
||||
continue
|
||||
}
|
||||
if(d.area.toDouble()<=25){
|
||||
continue
|
||||
}
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
if (newRegion) {
|
||||
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
def house_num = null
|
||||
if (houseNumStr) {
|
||||
house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "")
|
||||
}
|
||||
if (house_num) {
|
||||
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
|
||||
if (unitNum) {
|
||||
unitNum = unitNum.replace("单元", "")
|
||||
}
|
||||
def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num
|
||||
if(!keys.contains(key) && d.room_num) {
|
||||
keys << key
|
||||
def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num)
|
||||
if(!count){
|
||||
def dist = null
|
||||
if(d.match_city){
|
||||
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", d.district_name)
|
||||
udata.put("house_num", house_num)
|
||||
if(unitNum){
|
||||
udata.put("house_num_unit", unitNum)
|
||||
}
|
||||
udata.put("room_num", d.room_num)
|
||||
udata.put("house_area",d.area.toDouble())
|
||||
udata.put("status", 0)
|
||||
udata.put("from", "房谱网")
|
||||
if(d.floor_heigth && d.floor_heigth.isNumber()){
|
||||
udata.put("house_height", d.floor_heigth.toInteger())
|
||||
}
|
||||
if(d.floor && d.floor.isNumber()){
|
||||
udata.put("house_floor", d.floor.toInteger())
|
||||
}
|
||||
if(dist){
|
||||
udata.put("district_id", dist._id.toString())
|
||||
udata.put("name_in_seaweed", dist.name)
|
||||
}
|
||||
if(d.address){
|
||||
def address = AddressUtil.formatAddress(d.city, d.address)
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
HOUSE_LOUDONG_LINE_COLL.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", district_name)
|
||||
q.put("house_num", house_num)
|
||||
if(unitNum){
|
||||
q.put("unit_num", unitNum)
|
||||
}
|
||||
q.put("room_num", room_num)
|
||||
def count = HOUSE_LOUDONG_LINE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllLoundongByHouseNum(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
def map = [:]
|
||||
def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num
|
||||
if(d.unit_num){
|
||||
key = key +"##"+d.unit_num
|
||||
}
|
||||
def detailMap= map.get(key)
|
||||
if(!detailMap){
|
||||
detailMap = [:]
|
||||
map.put(key,detailMap)
|
||||
}
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
if(newRegion){
|
||||
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
def house_num = null
|
||||
if(houseNumStr){
|
||||
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
|
||||
}
|
||||
if(house_num){
|
||||
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
|
||||
if(unitNum){
|
||||
unitNum = unitNum.replace("单元","")
|
||||
}
|
||||
def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num
|
||||
if(unitNum){
|
||||
key = key +"##"+unitNum
|
||||
}
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num)
|
||||
println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}"
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
def areas = obj.areas
|
||||
if(!areas.contains(d.area)){
|
||||
areas << d.area
|
||||
|
||||
if(detailMap.height){
|
||||
def forms = obj.from
|
||||
if(!forms.contains("房谱网")){
|
||||
forms << "房谱网"
|
||||
}
|
||||
obj.m_date = new Date()
|
||||
def query = new BasicDBObject("_id", obj._id)
|
||||
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
|
||||
|
||||
}
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", d.district_name)
|
||||
udata.put("house_num", house_num)
|
||||
if(unitNum){
|
||||
udata.put("unit_num", unitNum)
|
||||
}
|
||||
if(d.address){
|
||||
def address = AddressUtil.formatAddress(d.city, d.address)
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
udata.put("areas",[d.area])
|
||||
udata.put("from",["房谱网"])
|
||||
if(d.floor_heigth && d.floor_heigth.toString().toInteger()){
|
||||
udata.put("floors", d.floor_heigth.toString().toInteger())
|
||||
}
|
||||
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("status", 1)
|
||||
LOUDONG_LINE_COLL.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addFdcHouseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000)
|
||||
def keys = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
if(!d.use_type){
|
||||
continue
|
||||
}
|
||||
if(!d.use_type.contains("住宅")){
|
||||
continue
|
||||
}
|
||||
def newRegion = d.match_region
|
||||
if(newRegion){
|
||||
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
def house_num = null
|
||||
if(houseNumStr){
|
||||
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
|
||||
}
|
||||
if(house_num){
|
||||
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num
|
||||
if(!keys.contains(key)) {
|
||||
keys << key
|
||||
def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num)
|
||||
if(!count){
|
||||
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
|
||||
def dist = null
|
||||
if(d.match_city){
|
||||
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
|
||||
}
|
||||
def floor = null
|
||||
if(d.room_num && d.room_num.contains("-")) {
|
||||
def arrs = d.room_num.split("-")
|
||||
floor = arrs[0]
|
||||
}
|
||||
Integer room = null
|
||||
Integer hall = null
|
||||
if(d.room_type && d.room_type.contains("室")){
|
||||
def roomStr = d.room_type.substring(0,d.room_type.indexOf("室"))
|
||||
if(roomStr){
|
||||
roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
|
||||
room = roomStr.toInteger()
|
||||
}
|
||||
|
||||
if(d.room_type.contains("厅")){
|
||||
def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅"))
|
||||
if(hallStr){
|
||||
hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
|
||||
hall = hallStr.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", d.name)
|
||||
udata.put("house_num", house_num)
|
||||
if(d.unit_num){
|
||||
udata.put("house_num_unit", d.unit_num)
|
||||
}
|
||||
udata.put("room_num", d.room_num)
|
||||
udata.put("house_area",d.area.toDouble())
|
||||
udata.put("status", 0)
|
||||
udata.put("from", "重庆网上房地产")
|
||||
udata.put("house_height", maxHeight)
|
||||
if(room){
|
||||
udata.put("room", room)
|
||||
}
|
||||
if(hall){
|
||||
udata.put("hall", hall)
|
||||
}
|
||||
if(floor && floor.isNumber()){
|
||||
udata.put("house_floor", floor.toInteger())
|
||||
}
|
||||
if(dist){
|
||||
udata.put("district_id", dist._id.toString())
|
||||
udata.put("name_in_seaweed", dist.name)
|
||||
}
|
||||
if(d.address){
|
||||
def address = AddressUtil.formatAddress(d.city, d.address)
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
HOUSE_LOUDONG_LINE_COLL.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String getMatchString(String text,String pattern){
|
||||
Pattern p=Pattern.compile(pattern);
|
||||
Matcher m=p.matcher(text);
|
||||
if(m.find()){
|
||||
return m.group(0)
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def addFdcBaseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongFdcConn.find(q).batchSize(5000)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
if(!d.match_region){
|
||||
continue
|
||||
}
|
||||
def newRegion = d.match_region
|
||||
|
||||
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
def house_num = null
|
||||
if(houseNumStr){
|
||||
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
|
||||
}
|
||||
if(house_num){
|
||||
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num
|
||||
if(d.unit_num) {
|
||||
key = key + "##" + d.unit_num
|
||||
|
||||
if (!keys.contains(key)) {
|
||||
keys << key
|
||||
def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num)
|
||||
println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}"
|
||||
|
||||
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
|
||||
if (obj) {
|
||||
|
||||
def areas = obj.areas
|
||||
if (!areas.contains(d.area)) {
|
||||
areas << d.area
|
||||
|
||||
def forms = obj.from
|
||||
if (!forms.contains("重庆网上房地产")) {
|
||||
forms << "重庆网上房地产"
|
||||
}
|
||||
obj.m_date = new Date()
|
||||
def query = new BasicDBObject("_id", obj._id)
|
||||
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
|
||||
|
||||
}
|
||||
} else {
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", d.name)
|
||||
udata.put("house_num", house_num)
|
||||
if (d.unit_num) {
|
||||
udata.put("unit_num", d.unit_num)
|
||||
}
|
||||
if (d.address) {
|
||||
def address = AddressUtil.formatAddress(d.city, d.address)
|
||||
if (address) {
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
udata.put("areas", [d.area])
|
||||
udata.put("from", ["重庆网上房地产"])
|
||||
udata.put("floors", maxHeight)
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("status", 1)
|
||||
LOUDONG_LINE_COLL.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findMaxHeight(city, name,house_num, unit_num){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("name",name)
|
||||
q.put("house_num_desc",house_num)
|
||||
q.put("unit_num",unit_num)
|
||||
def dbcursor = loudongFdcConn.find(q)
|
||||
def heights = []
|
||||
for(def h : dbcursor){
|
||||
if(h.nominal_floor && h.nominal_floor.isNumber()){
|
||||
def height = h.nominal_floor.toInteger()
|
||||
if(!heights.contains(height)){
|
||||
heights << height
|
||||
}
|
||||
}
|
||||
}
|
||||
if(heights){
|
||||
heights = heights.sort()
|
||||
return heights[heights.size()-1]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findLoudongOnLine(city, region, name, house_num, unit_num){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("region",region)
|
||||
q.put("name",name)
|
||||
q.put("house_num",house_num)
|
||||
if(unit_num){
|
||||
q.put("unit_num",unit_num)
|
||||
}
|
||||
def dbo = LOUDONG_LINE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
|
||||
|
||||
def stat(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -291,7 +651,51 @@ class CqfdcMatch {
|
||||
}
|
||||
|
||||
def addLoudong(){
|
||||
// def stat = statBaseLoudong()
|
||||
// println stat
|
||||
// findAllLoundongByHouseNum();
|
||||
def stat1 = statHouseLoudong()
|
||||
println stat1
|
||||
// addHouseLoudong()
|
||||
}
|
||||
|
||||
def addFdcLoudong(){
|
||||
// addFdcBaseLoudong()
|
||||
addFdcHouseLoudong()
|
||||
}
|
||||
|
||||
def statHouseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "重庆")
|
||||
def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q)
|
||||
def total = dbcursor.size()
|
||||
def ids = []
|
||||
for(def d : dbcursor){
|
||||
if(d.district_id && !ids.contains(d.district_id)){
|
||||
ids << d.district_id
|
||||
}
|
||||
}
|
||||
def map = [:]
|
||||
map.rows_total = total
|
||||
map.dist_count = ids.size()
|
||||
return map
|
||||
}
|
||||
|
||||
def statBaseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "重庆")
|
||||
def dbcursor = LOUDONG_LINE_COLL.find(q)
|
||||
def total = dbcursor.size()
|
||||
def ids = []
|
||||
for(def d : dbcursor){
|
||||
if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){
|
||||
ids << d.id_in_seaweed
|
||||
}
|
||||
}
|
||||
def map = [:]
|
||||
map.rows_total = total
|
||||
map.dist_count = ids.size()
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.fangjia.script.temp.shenjian.geo
|
||||
|
||||
import com.fangjia.script.gj.location.Point
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xh.InsidePolygon
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017)
|
||||
|
||||
collection.find(new BasicDBObject()).forEach({ obj ->
|
||||
if (obj.type == 'block') {
|
||||
if (obj.region instanceof String) {
|
||||
return;
|
||||
}
|
||||
BasicDBList reigon = (BasicDBList)obj.region
|
||||
int count = 0
|
||||
List<String> info = new ArrayList<>()
|
||||
for (String s : reigon) {
|
||||
info.add(s)
|
||||
}
|
||||
if (info.size() > 1) {
|
||||
boolean find = false
|
||||
for (String s : info) {
|
||||
if (find) {
|
||||
break
|
||||
}
|
||||
collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 ->
|
||||
String border = obj2.border
|
||||
String points = border.replaceAll(",", "-").replaceAll(";", "-")
|
||||
List<com.fangjia.script.looplineinfo.Point> pointList = InsidePolygon.createPoint(points);
|
||||
com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat)
|
||||
if (InsidePolygon.aboutZone(point, pointList)) {
|
||||
collection.update(new BasicDBObject("_id", obj._id),
|
||||
new BasicDBObject('$set', new BasicDBObject("region", obj2.name)));
|
||||
find = true;
|
||||
}
|
||||
})
|
||||
}
|
||||
} else {
|
||||
collection.update(new BasicDBObject("_id", obj._id),
|
||||
new BasicDBObject('$set', new BasicDBObject("region", info.get(0))));
|
||||
}
|
||||
}
|
||||
})
|
||||
+2
-2
@@ -128,10 +128,10 @@ class SeaweedUtils {
|
||||
*/
|
||||
static formatHouseholdCount(def household_count) {
|
||||
if (household_count && FieldValidatorUtils.checkValue(household_count) ) {
|
||||
if (Long.valueOf(household_count) < 50) {
|
||||
if (Integer.valueOf(household_count) < 50) {
|
||||
return "" // 户数
|
||||
}
|
||||
return household_count
|
||||
return household_count.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+28
-28
@@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{
|
||||
def static useDistKeys = []
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
|
||||
def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
|
||||
"保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
|
||||
"沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
|
||||
"石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
|
||||
"绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
|
||||
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
|
||||
"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
|
||||
"蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
|
||||
"兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362",
|
||||
"芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222",
|
||||
"南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
|
||||
@@ -192,37 +192,37 @@ class Esf58Crawl extends BaseBatch{
|
||||
void process(Object args) {
|
||||
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
|
||||
for(def city : citys){
|
||||
def str = cityList.get(city)
|
||||
for(def i=0;i<1;i++){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
if(districts){
|
||||
def arrs = str.split("\\|")
|
||||
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
distMap.put(city, districts)
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cityList.each{city, str->
|
||||
// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
|
||||
// for(def city : citys){
|
||||
// def str = cityList.get(city)
|
||||
// for(def i=0;i<1;i++){
|
||||
// if(citys.contains(city)){
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
// if(districts){
|
||||
// def arrs = str.split("\\|")
|
||||
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
// distMap.put(city, districts)
|
||||
//
|
||||
// addCrawlKeys(city);
|
||||
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
// }
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
// if(districts){
|
||||
// def arrs = str.split("\\|")
|
||||
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
// distMap.put(city, districts)
|
||||
// addCrawlKeys(city);
|
||||
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
cityList.each{city, str->
|
||||
for(def i=0;i<1;i++){
|
||||
// if(citys.contains(city)){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
if(districts){
|
||||
def arrs = str.split("\\|")
|
||||
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
distMap.put(city, districts)
|
||||
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202202);
|
||||
def count2 = findTotalCount(source, 202203);
|
||||
def count1 = findTotalCount(source, 202203);
|
||||
def count2 = findTotalCount(source, 202204);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,6 +229,93 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject searchLoudong(String id, String houseNum, String unitNum){
|
||||
try{
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("id", id);
|
||||
params.put("houseNumber", houseNum);
|
||||
params.put("unitNumber", unitNum);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
|
||||
println("${id} ${houseNum} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if(result){
|
||||
if (result.getInt("code") == 200) {
|
||||
return result;
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${id} ${houseNum}")
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
println("${id} ${houseNum} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){
|
||||
try{
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("id", id);
|
||||
params.put("houseNumber", houseNum);
|
||||
params.put("unitNumber", unitNum);
|
||||
params.put("roomNumber", roomNum);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
|
||||
println("${id} ${houseNum} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if(result){
|
||||
if (result.getInt("code") == 200) {
|
||||
return result;
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${id} ${houseNum}")
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
println("${id} ${houseNum} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject standard2Action(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("token", TOKEN);
|
||||
params.put("__refresh_cache","1")
|
||||
|
||||
String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8")
|
||||
println("${city} ${address} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if (result.getInt("code") == 0) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${city} ${address}")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${address} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject standardize(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
@@ -593,6 +680,90 @@ public class OpenApiClient {
|
||||
}
|
||||
|
||||
|
||||
static JSONObject facilitys(String city, String region, String name) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("city", city)
|
||||
params.put("region", region)
|
||||
params.put("name", name)
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static JSONObject searchPaimais(String city, String region, String name, limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("city", city)
|
||||
params.put("district", region)
|
||||
params.put("name", name)
|
||||
params.put("limit", "10")
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static JSONObject searchDealPrice(String id, String startDate, limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("id", id)
|
||||
params.put("startDate", startDate)
|
||||
params.put("limit", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static JSONObject houseList(String id, String startDate, limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("id", id)
|
||||
params.put("startTime", startDate)
|
||||
params.put("pageSize", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static JSONObject evaluaionList(String id, String startDate, limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("id", id)
|
||||
params.put("startDate", startDate)
|
||||
params.put("limit", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static JSONObject districtDetail(String city, String region, String name) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
|
||||
Reference in New Issue
Block a user