+3
-3
@@ -76,7 +76,7 @@ class ExportOracleData {
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -503,7 +503,7 @@ class ExportOracleData {
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -522,7 +522,7 @@ class ExportOracleData {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1)
|
||||
//def cityDataMap = [:]
|
||||
//for(def row : cityDatas){
|
||||
// if(row['D']){
|
||||
// cityDataMap.put(row['A']+"##"+row['B']+"##"+row['C'], row)
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据_1529.xlsx",{d->
|
||||
//
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// def key = d['A']+"##"+newRegion+"##"+d['C']
|
||||
// println key
|
||||
// def datas = cityDataMap.get(key)
|
||||
// if(datas){
|
||||
// d['F'] = datas['D']
|
||||
// d['G'] = datas['E']
|
||||
// d['H'] = datas['F']
|
||||
// d['I'] = datas['G']
|
||||
// d['J'] = datas['H']
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
|
||||
def city = "广州"
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(city, d['A'])
|
||||
// if(newRegion){
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null);
|
||||
// if(rMap){
|
||||
// d['C'] = rMap.region
|
||||
// d['D'] = rMap.name
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(city, d['E'])
|
||||
// if(newRegion){
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['F'], null);
|
||||
// if(rMap){
|
||||
// d['L'] = rMap.region
|
||||
// d['M'] = rMap.name
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",0, 1)
|
||||
//def idMaps = [:]
|
||||
//for(def d : datas){
|
||||
// if(d['L']){
|
||||
// idMaps.put(d['L']+"##"+d['M'], d)
|
||||
// }
|
||||
//}
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
// if(d['C']){
|
||||
// def key = d['C']+"##"+d['D']
|
||||
// def rows = idMaps.get(key)
|
||||
// if(rows){
|
||||
// d['E'] = rows['D']
|
||||
// d['F'] = rows['E']
|
||||
// d['G'] = rows['F']
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district")
|
||||
// if(obj){
|
||||
// def year = CommonUtils.getDistrictAge(obj)
|
||||
// if(year){
|
||||
// d['H'] = year
|
||||
// }
|
||||
// if(obj.alias){
|
||||
// d['K'] = obj.alias.join(" ")
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
|
||||
def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
|
||||
def scoreMap = [:]
|
||||
for(def d : data1s){
|
||||
scoreMap.put(d['B'], d)
|
||||
}
|
||||
|
||||
def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
|
||||
def levelMap = [:]
|
||||
for(def d : data2s){
|
||||
levelMap.put(d['E'], d)
|
||||
}
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
if(d['E']){
|
||||
def row = scoreMap.get(d['E'])
|
||||
if(row){
|
||||
d['I'] = row['G']
|
||||
d['J'] = row['H']
|
||||
d['K'] = row['I']
|
||||
d['L'] = row['J']
|
||||
d['M'] = row['K']
|
||||
d['N'] = row['L']
|
||||
d['O'] = row['M']
|
||||
}
|
||||
|
||||
def data = levelMap.get(d['E'])
|
||||
if(data){
|
||||
d['P'] = data['G']
|
||||
d['Q'] = data['H']
|
||||
d['R'] = data['I']
|
||||
d['S'] = data['J']
|
||||
d['T'] = data['K']
|
||||
d['U'] = data['L']
|
||||
d['V'] = data['M']
|
||||
}
|
||||
}
|
||||
},0)
|
||||
+47
-13
@@ -19,6 +19,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class MatchMapData {
|
||||
|
||||
def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol();
|
||||
@@ -28,12 +30,12 @@ class MatchMapData {
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchMapData()
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.exportHouse()
|
||||
task.addDistrictLevel()
|
||||
// task.addDistrictLevel()
|
||||
|
||||
}
|
||||
|
||||
@@ -135,16 +137,22 @@ class MatchMapData {
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"合肥",
|
||||
def files = ["上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"]
|
||||
//"广州",
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def files = ["合肥","广州","上海","北京","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"]
|
||||
for(def f : files){
|
||||
def city = f.replace("-2","")
|
||||
def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365)
|
||||
def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear))
|
||||
def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend)
|
||||
|
||||
def datas = []
|
||||
datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型"]
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def priceDatas = addPriceData(dists, priceMap);
|
||||
def levelMap = calcuteLevel(priceDatas);
|
||||
|
||||
def datas = new CopyOnWriteArrayList()
|
||||
datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型","评级","均价"]
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d->
|
||||
def regionCode = null
|
||||
if(d['E']){
|
||||
@@ -192,13 +200,23 @@ class MatchMapData {
|
||||
d['N'] = prc.toInteger()
|
||||
}
|
||||
d['O'] = dist.estate_type2
|
||||
|
||||
|
||||
def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name)
|
||||
if(dist.name == "绿苑公寓"){
|
||||
println dbo
|
||||
}
|
||||
if(dbo){
|
||||
d['P'] = dbo.level
|
||||
d['Q'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d['M'] = "区域无法标准化"
|
||||
}
|
||||
}
|
||||
|
||||
datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O']]
|
||||
datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O'], d['P'], d['Q']]
|
||||
|
||||
},0)
|
||||
|
||||
@@ -385,12 +403,12 @@ class MatchMapData {
|
||||
}
|
||||
|
||||
def addDistrictLevel(){
|
||||
//
|
||||
def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"]
|
||||
|
||||
def dataMap = null
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
|
||||
for(def fn : files){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend)
|
||||
@@ -399,15 +417,28 @@ class MatchMapData {
|
||||
|
||||
dataMap = getDataMap(fn);
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\评级10级\\"+fn+".xlsx",{d->
|
||||
|
||||
def region = dataMap.get(d['C'])
|
||||
if(region){
|
||||
def newRegion = RegionData.formatRegion(fn, region)
|
||||
if(newRegion){
|
||||
def level = levelMap.get( fn+"##"+newRegion+"##"+d['D'])
|
||||
if(level){
|
||||
d['J'] = level
|
||||
def dist = CommonDatautils.findDistrict(fn, newRegion, d['D'],"district")
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(fn, newRegion, d['D'],"district")
|
||||
if(!dist){
|
||||
def rets = CommonDatautils.findDistrictByAlias(fn, newRegion, d['D'],"district")
|
||||
if(rets){
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if(dist){
|
||||
def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name)
|
||||
if(dbo){
|
||||
d['J'] = dbo.level
|
||||
d['K'] = dbo.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -472,7 +503,10 @@ class MatchMapData {
|
||||
}
|
||||
def dataMap = [:]
|
||||
for(def d : datas){
|
||||
dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level)
|
||||
if(d.name=="绿苑公寓"){
|
||||
println d
|
||||
}
|
||||
dataMap.put(d.city+"##"+d.region+"##"+d.name, d)
|
||||
}
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
+1
-1
@@ -105,7 +105,7 @@ class SSHHelper {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20211210"));
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -705,7 +705,7 @@ class CommonDatautils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","长隆小区")
|
||||
// q.put("name","吴中路9弄小区")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -829,6 +829,8 @@ class CommonDatautils {
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("name_cat", 1);
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
|
||||
+420
@@ -0,0 +1,420 @@
|
||||
package com.fangjia.script.gj.level
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.util.DateUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc"
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictIndexBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["上海"]//CommonDatautils.findAllCity();
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
|
||||
calcute(city, dists)
|
||||
}
|
||||
}
|
||||
|
||||
def calcute(city, dists){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def costDatas = getCostByCity(city)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def datas = new CopyOnWriteArrayList()
|
||||
datas << ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(d)
|
||||
def avg_charge = null
|
||||
def rank = null
|
||||
if (list) {
|
||||
list = sortByDistance(d, list);
|
||||
avg_charge = gtAvgEstateCharge(list)
|
||||
rank = getPriceInRegionRank(d, list, priceMap)
|
||||
}
|
||||
def total_price = getHousePrice(d, dists, priceMap);
|
||||
def avg_cost = getAvgCost(d, costDatas)
|
||||
|
||||
def total_price_index = 0
|
||||
if (total_price) {
|
||||
total_price_index = 286.0134196 * (total_price - 531.2615913) / 313.3693508
|
||||
}
|
||||
def avg_charge_index = 0
|
||||
if (avg_charge) {
|
||||
avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241
|
||||
}
|
||||
|
||||
def rank_index = 0
|
||||
if (rank) {
|
||||
rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636
|
||||
}
|
||||
|
||||
def avg_cost_index = 0
|
||||
if (avg_cost) {
|
||||
avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959
|
||||
}
|
||||
def potentiality = avg_cost_index + 4099.978248
|
||||
potentiality = DataHandleUtils.getDoubleRound(2, potentiality)
|
||||
def map = [:]
|
||||
map.id = new ObjectId().toString()
|
||||
map.city = d.city
|
||||
map.region = d.region
|
||||
map.name = d.name
|
||||
map.avg_total_price = total_price
|
||||
map.avg_estate_charge = avg_charge
|
||||
map.price_rank = rank
|
||||
map.avg_cost = avg_cost
|
||||
map.comp_value = potentiality
|
||||
map.month = "2022-01"
|
||||
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
|
||||
|
||||
def newUrl = targetUrl +"/"+map.id
|
||||
httpSaveData(newUrl, map)
|
||||
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown();
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await();
|
||||
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
|
||||
}
|
||||
|
||||
def getHousePrice(d, dists, priceMap){
|
||||
def areas = findHouse(d, 20200101, 20220101)
|
||||
def avg_area = null
|
||||
if(areas){
|
||||
avg_area = areas.sum()/areas.size();
|
||||
}else{
|
||||
if(d.finish_building_date){
|
||||
def year = CommonUtils.getBuildYear(d.finish_building_date)
|
||||
if(year){
|
||||
def startYear = (year.toString().substring(0,3)+"0").toInteger()
|
||||
def endYear = startYear + 10
|
||||
|
||||
def selectDists = []
|
||||
for(def ad : dists){
|
||||
if(d.region == ad.region){
|
||||
def adYear = CommonUtils.getBuildYear(ad.finish_building_date)
|
||||
if(adYear){
|
||||
adYear = adYear.toInteger()
|
||||
if(adYear>=startYear && adYear<=endYear){
|
||||
selectDists << ad
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(selectDists){
|
||||
def allAreas = []
|
||||
for(def sd : selectDists){
|
||||
def areaDatas = findHouse(sd, 20190101, 20220101)
|
||||
allAreas.addAll(areaDatas)
|
||||
}
|
||||
avg_area = allAreas.sum()/allAreas.size();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(avg_area){
|
||||
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
def total_price = (avg_price * avg_area)/10000.0
|
||||
def total = DataHandleUtils.getDoubleRound(2, total_price)
|
||||
return total
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def gtAvgEstateCharge(list){
|
||||
def rets = []
|
||||
for(def i=0;i<list.size();i++){
|
||||
def nb = list[i]
|
||||
def detail = CommonDatautils.findDistrictFieldById(nb._id.toString(), ["estate_charge"])
|
||||
if(detail && detail.estate_charge){
|
||||
rets << detail.estate_charge.toDouble()
|
||||
if(rets.size()>=3){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
def avg_charge = rets.sum()/rets.size();
|
||||
avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge)
|
||||
return avg_charge;
|
||||
}
|
||||
|
||||
def getPriceInRegionRank(d, nbLists, priceMap){
|
||||
if(nbLists){
|
||||
if(nbLists.size()==1){
|
||||
return 0.5;
|
||||
}else{
|
||||
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
def count = 0
|
||||
if(avg_price){
|
||||
for(def nb : nbLists){
|
||||
def price = priceMap.get(nb.city+"##"+nb.region+"##"+nb.name)
|
||||
if(price > avg_price){
|
||||
count ++
|
||||
}
|
||||
}
|
||||
}
|
||||
def pre = (nbLists.size() - count)/nbLists.size()
|
||||
pre = DataHandleUtils.getDoubleRound(2, pre)
|
||||
return pre;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getCostByCity(city){
|
||||
def datas = queryPoi(city)
|
||||
return datas
|
||||
}
|
||||
|
||||
def sortByDistance(d, list){
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
}
|
||||
//排序
|
||||
Collections.sort(list, new Comparator() {
|
||||
@Override
|
||||
int compare(Object o1, Object o2) {return o1.distance-o2.distance;}
|
||||
})
|
||||
return list;
|
||||
}
|
||||
|
||||
def getAvgCost(d, costDatas){
|
||||
if(costDatas){
|
||||
def costs = []
|
||||
for(def c : costDatas){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, c.lat, c.lng)
|
||||
if(distance <= 1000){
|
||||
costs << c.cost
|
||||
}
|
||||
}
|
||||
if(costs){
|
||||
def avg_cost = costs.sum()/costs.size()
|
||||
avg_cost = DataHandleUtils.getDoubleRound(2, avg_cost)
|
||||
return avg_cost
|
||||
}
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def queryPoi(def city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode,business.keytag as keytag,business.cost as cost FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "市'")
|
||||
sqlBuffer.append(" and business.cost>20 and business.cost<500")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def datas = []
|
||||
def tags = ["法国菜","东南亚菜","日本料理","火锅","海鲜","自助餐","西餐","浙菜","茶餐厅","酒吧","烤肉","潮州菜","云贵菜","韩国料理","中餐","特色火锅","西北菜","串串香","农家乐","烧烤","茶座","上海菜","意大利菜","素菜馆","蛋糕店","东北菜","湘菜","粤菜","咖啡厅","轰趴馆","清真菜","北京菜","川菜","餐饮服务,餐饮相关场所,餐饮相关"]
|
||||
def index = 0
|
||||
while (rs.next()) {
|
||||
index++
|
||||
def tag = rs.getString("keytag")
|
||||
if(tags.contains(tag)){
|
||||
def dMap = [:]
|
||||
dMap.cost = rs.getInt("cost")
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
dMap.lng = location.x
|
||||
dMap.lat = location.y
|
||||
}
|
||||
datas << dMap
|
||||
}else{
|
||||
println tag
|
||||
}
|
||||
}
|
||||
println index
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
def findHouse(def dist, def startDate, def endDate){
|
||||
def nameQuery = getAllName(dist);
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT area FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+dist.city+"'")
|
||||
sqlBuffer.append(" and region = '"+dist.region+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
sqlBuffer.append(" and (" + nameQuery + ")");
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def areas = []
|
||||
while (rs.next()){
|
||||
areas << rs.getDouble("area")
|
||||
}
|
||||
return areas;
|
||||
}
|
||||
|
||||
def getAllName(dist){
|
||||
Set<String> aliasNames = new LinkedHashSet<String>();
|
||||
aliasNames.add(dist.name);
|
||||
if (dist != null && dist.alias != null) {
|
||||
aliasNames.addAll(dist.alias);
|
||||
}
|
||||
|
||||
List<String> namePars = new ArrayList<String>();
|
||||
if(aliasNames!=null && aliasNames.size()>0) {
|
||||
for(String name : aliasNames){
|
||||
if(name && name.trim().length() > 1){
|
||||
namePars.add("district_name ='" + name + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(namePars.size() > 0){
|
||||
return namePars.join(" OR ")
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// def updateHouseMatchDistrictId(def id, def match_id){
|
||||
// def header = [:]
|
||||
// header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString()
|
||||
// String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
// def body = """
|
||||
// {
|
||||
// "script": {
|
||||
// "source": "ctx._source.match_seaweed_id='${match_id}'",
|
||||
// "lang": "painless"
|
||||
// },
|
||||
// "query": {
|
||||
// "term": {
|
||||
// "_id": "${id}"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// """
|
||||
// HttpRequest request = new HttpRequest()
|
||||
// String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// println s
|
||||
// }
|
||||
|
||||
|
||||
|
||||
def httpSaveData(def newUrl,def params){
|
||||
def header = [:]
|
||||
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
|
||||
doPost(newUrl,header, params, true, "utf-8")
|
||||
}
|
||||
|
||||
|
||||
def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||
try{
|
||||
URL url = new URL(targetUrl);
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setDoOutput(true);
|
||||
conn.setUseCaches(false);
|
||||
conn.setConnectTimeout(5000);
|
||||
conn.setReadTimeout(5000);
|
||||
conn.setRequestProperty("User-Agent", UserAgents.get());
|
||||
String param = null;
|
||||
BufferedWriter mWriter = null;
|
||||
InputStreamReader inStream = null;
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
conn.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
} else {
|
||||
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
|
||||
}
|
||||
|
||||
if (isJson) {
|
||||
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
|
||||
param = JacksonUtil.beanToJson(params)
|
||||
}
|
||||
|
||||
conn.connect();
|
||||
|
||||
//println param.toString()
|
||||
//输出流
|
||||
if (StringUtils.isNotEmpty(param)) {
|
||||
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
|
||||
mWriter.write(param);
|
||||
mWriter.flush();
|
||||
mWriter.close();
|
||||
}
|
||||
if (conn.getHeaderField("Content-Encoding") != null
|
||||
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
|
||||
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
|
||||
} else {
|
||||
inStream = new InputStreamReader(conn.getInputStream(), charset);
|
||||
}
|
||||
char[] bf = new char[1024];
|
||||
int len;
|
||||
while ((len = inStream.read(bf)) != -1) {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
+54
-11
@@ -38,9 +38,10 @@ class Esf58Crawl extends BaseBatch{
|
||||
def static useDistKeys = []
|
||||
static def totalCount = new AtomicLong()
|
||||
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102",
|
||||
// "上海":"sh|2", "北京":"bj|1","天津":"tj|18",
|
||||
// "青岛":"qd|122",
|
||||
// "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158",
|
||||
def cityList = ["青岛":"qd|122","南京":"nj|172",
|
||||
// "上海":"sh|2",
|
||||
def cityList = [ "北京":"bj|1","天津":"tj|18","南京":"nj|172",
|
||||
"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463",
|
||||
"南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
|
||||
"呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
|
||||
@@ -261,7 +262,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
def newUrl = url+"?sort=param62_asc&key="+URLEncoder.encode(keyword, "utf-8")
|
||||
def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET);
|
||||
if(document){
|
||||
Elements elements = JQueryUtils.select([".list-cell a"], document)
|
||||
Elements elements = JQueryUtils.select([".list-cell .li-row"], document)
|
||||
if(elements){
|
||||
addSearchContent(city, url, keyword);
|
||||
}else{
|
||||
@@ -279,9 +280,11 @@ class Esf58Crawl extends BaseBatch{
|
||||
def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET);
|
||||
println "[${totalCount.get()}]关键字【${keyword}】 页:"+ newUrl
|
||||
if(document){
|
||||
Elements elements = JQueryUtils.select([".list-cell a"], document)
|
||||
if(elements){
|
||||
saveDetailContent(city, document);
|
||||
Elements infoElements = JQueryUtils.select([".list-cell .li-info"], document)
|
||||
Elements priceElements = JQueryUtils.select([".list-cell .li-side"], document)
|
||||
if(infoElements){
|
||||
// saveDetailContent(city, document);
|
||||
saveLists(city, infoElements, priceElements)
|
||||
}else{
|
||||
flag = false;
|
||||
}
|
||||
@@ -289,6 +292,46 @@ class Esf58Crawl extends BaseBatch{
|
||||
flag = false;
|
||||
}
|
||||
page++;
|
||||
if(page>50){
|
||||
flag = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def saveLists(city, infoElements, priceElements){
|
||||
for(def i=0;i<infoElements.size();i++){
|
||||
def e = infoElements.get(i)
|
||||
def priceEle = priceElements.get(i)
|
||||
def name = e.select(".li-title").text()
|
||||
def avg_price = priceEle.select(".community-price strong").text()
|
||||
def addressStr = e.select(".props span:eq(2)").text()
|
||||
def region = null
|
||||
def address = null
|
||||
def href = e.select(".li-row").attr("href")
|
||||
if(addressStr){
|
||||
region = addressStr.toString().substring(0, addressStr.indexOf("-")).replace(" ","")
|
||||
address = addressStr.toString().substring(addressStr.toString().lastIndexOf("-")+1, addressStr.toString().length())
|
||||
}
|
||||
|
||||
if(avg_price){
|
||||
avg_price = avg_price.toInteger()
|
||||
}else{
|
||||
avg_price = 0
|
||||
}
|
||||
def obj = findByParams(city, region, name);
|
||||
if(obj){
|
||||
DBObject udata2 = new BasicDBObject()
|
||||
udata2.put("avg_price", avg_price);
|
||||
udata2.put("c_date", new Date());
|
||||
println "[update] "+city + " "+ name
|
||||
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
|
||||
}else{
|
||||
|
||||
println "[save] "+city + " "+ name
|
||||
saveThirdParty(["url":href,"city":city,"region":region,"block":null,"name":name,"address": address, "avg_price":avg_price]);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,18 +353,18 @@ class Esf58Crawl extends BaseBatch{
|
||||
def detailDocument = CrawalUtil.getDocumentByABuYun(href, ERROR_MSG, CHARSET, "ee");
|
||||
if(detailDocument){
|
||||
def name = JQueryUtils.selectText(".title", detailDocument)
|
||||
def navs = JQueryUtils.selectText(".breadcrumb-nav", detailDocument)
|
||||
def navs = JQueryUtils.selectText(".crumbs-middle", detailDocument)
|
||||
def info
|
||||
if(navs){
|
||||
info = navs.split(">")
|
||||
}
|
||||
def avg_price = JQueryUtils.selectText(".price", detailDocument);
|
||||
def avg_price = JQueryUtils.selectText(".average", detailDocument);
|
||||
def region;
|
||||
def addr = JQueryUtils.selectText("td:contains(详细地址)+td", detailDocument);
|
||||
def addr = JQueryUtils.selectText(".column-1:contains(小区地址) .value", detailDocument);
|
||||
if(info){
|
||||
if(info.size()>=4){
|
||||
city = info[2].replace("小区","").trim()
|
||||
region = info[3].replace("小区","").trim()
|
||||
city = info[1].replace("小区","").trim()
|
||||
region = info[2].replace("小区","").trim()
|
||||
}
|
||||
}
|
||||
if(!addr){
|
||||
|
||||
+2
-1
@@ -17,7 +17,8 @@ class SynchDistrictPrice {
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
def sourceMap = ["fang.com":"搜房","cityhouse.cn":"城市房产","5i5j.com":"我爱我家"]
|
||||
def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多",
|
||||
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
Reference in New Issue
Block a user