Merge branch 'script_gj' into 'master'

update

See merge request !239
This commit is contained in:
2022-01-10 11:20:26 +08:00
8 changed files with 657 additions and 30 deletions
@@ -76,7 +76,7 @@ class ExportOracleData {
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
}
@@ -503,7 +503,7 @@ class ExportOracleData {
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
}
@@ -522,7 +522,7 @@ class ExportOracleData {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
}
@@ -0,0 +1,127 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1)
//def cityDataMap = [:]
//for(def row : cityDatas){
// if(row['D']){
// cityDataMap.put(row['A']+"##"+row['B']+"##"+row['C'], row)
// }
//}
//
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据_1529.xlsx",{d->
//
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// def key = d['A']+"##"+newRegion+"##"+d['C']
// println key
// def datas = cityDataMap.get(key)
// if(datas){
// d['F'] = datas['D']
// d['G'] = datas['E']
// d['H'] = datas['F']
// d['I'] = datas['G']
// d['J'] = datas['H']
//
// }
//},0)
def city = "广州"
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
// def newRegion = RegionData.formatRegion(city, d['A'])
// if(newRegion){
// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null);
// if(rMap){
// d['C'] = rMap.region
// d['D'] = rMap.name
// }
// }
//
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",{d->
// def newRegion = RegionData.formatRegion(city, d['E'])
// if(newRegion){
// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['F'], null);
// if(rMap){
// d['L'] = rMap.region
// d['M'] = rMap.name
// }
// }
//},0)
//def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",0, 1)
//def idMaps = [:]
//for(def d : datas){
// if(d['L']){
// idMaps.put(d['L']+"##"+d['M'], d)
// }
//}
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
// if(d['C']){
// def key = d['C']+"##"+d['D']
// def rows = idMaps.get(key)
// if(rows){
// d['E'] = rows['D']
// d['F'] = rows['E']
// d['G'] = rows['F']
// }
// }
//},0)
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district")
// if(obj){
// def year = CommonUtils.getDistrictAge(obj)
// if(year){
// d['H'] = year
// }
// if(obj.alias){
// d['K'] = obj.alias.join(" ")
// }
//
// }
//},0)
def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
def scoreMap = [:]
for(def d : data1s){
scoreMap.put(d['B'], d)
}
def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
def levelMap = [:]
for(def d : data2s){
levelMap.put(d['E'], d)
}
ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
if(d['E']){
def row = scoreMap.get(d['E'])
if(row){
d['I'] = row['G']
d['J'] = row['H']
d['K'] = row['I']
d['L'] = row['J']
d['M'] = row['K']
d['N'] = row['L']
d['O'] = row['M']
}
def data = levelMap.get(d['E'])
if(data){
d['P'] = data['G']
d['Q'] = data['H']
d['R'] = data['I']
d['S'] = data['J']
d['T'] = data['K']
d['U'] = data['L']
d['V'] = data['M']
}
}
},0)
@@ -19,6 +19,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
class MatchMapData {
def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol();
@@ -28,12 +30,12 @@ class MatchMapData {
static main(args) {
def task = new MatchMapData()
// task.run()
task.run()
// task.run1()
// task.run2()
// task.run3()
// task.exportHouse()
task.addDistrictLevel()
// task.addDistrictLevel()
}
@@ -135,16 +137,22 @@ class MatchMapData {
}
def run(){
//"合肥",
def files = ["上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"]
//"广州",
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def files = ["合肥","广州","上海","北京","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"]
for(def f : files){
def city = f.replace("-2","")
def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365)
def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear))
def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend)
def datas = []
datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型"]
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def priceDatas = addPriceData(dists, priceMap);
def levelMap = calcuteLevel(priceDatas);
def datas = new CopyOnWriteArrayList()
datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型","评级","均价"]
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d->
def regionCode = null
if(d['E']){
@@ -192,13 +200,23 @@ class MatchMapData {
d['N'] = prc.toInteger()
}
d['O'] = dist.estate_type2
def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name)
if(dist.name == "绿苑公寓"){
println dbo
}
if(dbo){
d['P'] = dbo.level
d['Q'] = dbo.avg_price
}
}
}else{
d['M'] = "区域无法标准化"
}
}
datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O']]
datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O'], d['P'], d['Q']]
},0)
@@ -385,12 +403,12 @@ class MatchMapData {
}
def addDistrictLevel(){
//
def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"]
def dataMap = null
def weekend = DateUtils.getFirstDayOfWeek(new Date());
for(def fn : files){
def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend)
@@ -399,15 +417,28 @@ class MatchMapData {
dataMap = getDataMap(fn);
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\评级10级\\"+fn+".xlsx",{d->
def region = dataMap.get(d['C'])
if(region){
def newRegion = RegionData.formatRegion(fn, region)
if(newRegion){
def level = levelMap.get( fn+"##"+newRegion+"##"+d['D'])
if(level){
d['J'] = level
def dist = CommonDatautils.findDistrict(fn, newRegion, d['D'],"district")
if(!dist){
dist = CommonDatautils.findSeaweedByMappingAlias(fn, newRegion, d['D'],"district")
if(!dist){
def rets = CommonDatautils.findDistrictByAlias(fn, newRegion, d['D'],"district")
if(rets){
dist = rets[0]
}
}
}
if(dist){
def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name)
if(dbo){
d['J'] = dbo.level
d['K'] = dbo.avg_price
}
}
}
}
@@ -472,7 +503,10 @@ class MatchMapData {
}
def dataMap = [:]
for(def d : datas){
dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level)
if(d.name=="绿苑公寓"){
println d
}
dataMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
return dataMap;
}
@@ -105,7 +105,7 @@ class SSHHelper {
public static void main(String[] args) {
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20211210"));
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110"));
}
@@ -705,7 +705,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","长隆小区")
// q.put("name","吴中路9弄小区")
DBObject keys = new BasicDBObject();
if(fields){
@@ -829,6 +829,8 @@ class CommonDatautils {
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("name_cat", 1);
keys.put("name_cat", 1);
keys.put("name_cat", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -0,0 +1,420 @@
package com.fangjia.script.gj.level
import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.commons.httpclient.util.DateUtil
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.zip.GZIPInputStream
class DistrictIndexBatch extends BaseBatch{
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new DistrictIndexBatch(), args);
}
@Override
public void process(Object args) {
run();
}
def run(){
def citys = ["上海"]//CommonDatautils.findAllCity();
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
calcute(city, dists)
}
}
def calcute(city, dists){
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def costDatas = getCostByCity(city)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def datas = new CopyOnWriteArrayList()
datas << ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
List<DBObject> list = DbUtils.findAroundDistrict(d)
def avg_charge = null
def rank = null
if (list) {
list = sortByDistance(d, list);
avg_charge = gtAvgEstateCharge(list)
rank = getPriceInRegionRank(d, list, priceMap)
}
def total_price = getHousePrice(d, dists, priceMap);
def avg_cost = getAvgCost(d, costDatas)
def total_price_index = 0
if (total_price) {
total_price_index = 286.0134196 * (total_price - 531.2615913) / 313.3693508
}
def avg_charge_index = 0
if (avg_charge) {
avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241
}
def rank_index = 0
if (rank) {
rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636
}
def avg_cost_index = 0
if (avg_cost) {
avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959
}
def potentiality = avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(2, potentiality)
def map = [:]
map.id = new ObjectId().toString()
map.city = d.city
map.region = d.region
map.name = d.name
map.avg_total_price = total_price
map.avg_estate_charge = avg_charge
map.price_rank = rank
map.avg_cost = avg_cost
map.comp_value = potentiality
map.month = "2022-01"
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
def newUrl = targetUrl +"/"+map.id
httpSaveData(newUrl, map)
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown();
}
}
})
}
latch.await();
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
}
def getHousePrice(d, dists, priceMap){
def areas = findHouse(d, 20200101, 20220101)
def avg_area = null
if(areas){
avg_area = areas.sum()/areas.size();
}else{
if(d.finish_building_date){
def year = CommonUtils.getBuildYear(d.finish_building_date)
if(year){
def startYear = (year.toString().substring(0,3)+"0").toInteger()
def endYear = startYear + 10
def selectDists = []
for(def ad : dists){
if(d.region == ad.region){
def adYear = CommonUtils.getBuildYear(ad.finish_building_date)
if(adYear){
adYear = adYear.toInteger()
if(adYear>=startYear && adYear<=endYear){
selectDists << ad
}
}
}
}
if(selectDists){
def allAreas = []
for(def sd : selectDists){
def areaDatas = findHouse(sd, 20190101, 20220101)
allAreas.addAll(areaDatas)
}
avg_area = allAreas.sum()/allAreas.size();
}
}
}
}
if(avg_area){
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
def total_price = (avg_price * avg_area)/10000.0
def total = DataHandleUtils.getDoubleRound(2, total_price)
return total
}
return null
}
def gtAvgEstateCharge(list){
def rets = []
for(def i=0;i<list.size();i++){
def nb = list[i]
def detail = CommonDatautils.findDistrictFieldById(nb._id.toString(), ["estate_charge"])
if(detail && detail.estate_charge){
rets << detail.estate_charge.toDouble()
if(rets.size()>=3){
break;
}
}
}
def avg_charge = rets.sum()/rets.size();
avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge)
return avg_charge;
}
def getPriceInRegionRank(d, nbLists, priceMap){
if(nbLists){
if(nbLists.size()==1){
return 0.5;
}else{
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
def count = 0
if(avg_price){
for(def nb : nbLists){
def price = priceMap.get(nb.city+"##"+nb.region+"##"+nb.name)
if(price > avg_price){
count ++
}
}
}
def pre = (nbLists.size() - count)/nbLists.size()
pre = DataHandleUtils.getDoubleRound(2, pre)
return pre;
}
}
return null;
}
def getCostByCity(city){
def datas = queryPoi(city)
return datas
}
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
}
//排序
Collections.sort(list, new Comparator() {
@Override
int compare(Object o1, Object o2) {return o1.distance-o2.distance;}
})
return list;
}
def getAvgCost(d, costDatas){
if(costDatas){
def costs = []
for(def c : costDatas){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, c.lat, c.lng)
if(distance <= 1000){
costs << c.cost
}
}
if(costs){
def avg_cost = costs.sum()/costs.size()
avg_cost = DataHandleUtils.getDoubleRound(2, avg_cost)
return avg_cost
}
}
return null
}
def queryPoi(def city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode,business.keytag as keytag,business.cost as cost FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "市'")
sqlBuffer.append(" and business.cost>20 and business.cost<500")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def datas = []
def tags = ["法国菜","东南亚菜","日本料理","火锅","海鲜","自助餐","西餐","浙菜","茶餐厅","酒吧","烤肉","潮州菜","云贵菜","韩国料理","中餐","特色火锅","西北菜","串串香","农家乐","烧烤","茶座","上海菜","意大利菜","素菜馆","蛋糕店","东北菜","湘菜","粤菜","咖啡厅","轰趴馆","清真菜","北京菜","川菜","餐饮服务,餐饮相关场所,餐饮相关"]
def index = 0
while (rs.next()) {
index++
def tag = rs.getString("keytag")
if(tags.contains(tag)){
def dMap = [:]
dMap.cost = rs.getInt("cost")
def location = rs.getObject("location")
if(location){
dMap.lng = location.x
dMap.lat = location.y
}
datas << dMap
}else{
println tag
}
}
println index
return datas;
}
def findHouse(def dist, def startDate, def endDate){
def nameQuery = getAllName(dist);
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT area FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+dist.city+"'")
sqlBuffer.append(" and region = '"+dist.region+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
sqlBuffer.append(" and (" + nameQuery + ")");
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def areas = []
while (rs.next()){
areas << rs.getDouble("area")
}
return areas;
}
def getAllName(dist){
Set<String> aliasNames = new LinkedHashSet<String>();
aliasNames.add(dist.name);
if (dist != null && dist.alias != null) {
aliasNames.addAll(dist.alias);
}
List<String> namePars = new ArrayList<String>();
if(aliasNames!=null && aliasNames.size()>0) {
for(String name : aliasNames){
if(name && name.trim().length() > 1){
namePars.add("district_name ='" + name + "'");
}
}
}
if(namePars.size() > 0){
return namePars.join(" OR ")
}
return null;
}
// def updateHouseMatchDistrictId(def id, def match_id){
// def header = [:]
// header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString()
// String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
// def body = """
// {
// "script": {
// "source": "ctx._source.match_seaweed_id='${match_id}'",
// "lang": "painless"
// },
// "query": {
// "term": {
// "_id": "${id}"
// }
// }
// }
// """
// HttpRequest request = new HttpRequest()
// String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// println s
// }
def httpSaveData(def newUrl,def params){
def header = [:]
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
doPost(newUrl,header, params, true, "utf-8")
}
def doPost(def targetUrl, def headers, def params, def isJson, def charset){
try{
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setUseCaches(false);
conn.setConnectTimeout(5000);
conn.setReadTimeout(5000);
conn.setRequestProperty("User-Agent", UserAgents.get());
String param = null;
BufferedWriter mWriter = null;
InputStreamReader inStream = null;
StringBuilder buffer = new StringBuilder();
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
conn.setRequestProperty(head, headers.get(head));
}
} else {
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
}
if (isJson) {
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
param = JacksonUtil.beanToJson(params)
}
conn.connect();
//println param.toString()
//输出流
if (StringUtils.isNotEmpty(param)) {
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
mWriter.write(param);
mWriter.flush();
mWriter.close();
}
if (conn.getHeaderField("Content-Encoding") != null
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
} else {
inStream = new InputStreamReader(conn.getInputStream(), charset);
}
char[] bf = new char[1024];
int len;
while ((len = inStream.read(bf)) != -1) {
buffer.append(bf, 0, len);
}
} catch (Exception e) {
e.printStackTrace()
}
return null;
}
}
@@ -38,9 +38,10 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102",
// "上海":"sh|2", "北京":"bj|1","天津":"tj|18",
// "青岛":"qd|122",
// "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158",
def cityList = ["青岛":"qd|122","南京":"nj|172",
// "上海":"sh|2",
def cityList = [ "北京":"bj|1","天津":"tj|18","南京":"nj|172",
"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463",
"南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
"呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",
@@ -261,7 +262,7 @@ class Esf58Crawl extends BaseBatch{
def newUrl = url+"?sort=param62_asc&key="+URLEncoder.encode(keyword, "utf-8")
def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET);
if(document){
Elements elements = JQueryUtils.select([".list-cell a"], document)
Elements elements = JQueryUtils.select([".list-cell .li-row"], document)
if(elements){
addSearchContent(city, url, keyword);
}else{
@@ -279,9 +280,11 @@ class Esf58Crawl extends BaseBatch{
def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET);
println "[${totalCount.get()}]关键字【${keyword}】 页:"+ newUrl
if(document){
Elements elements = JQueryUtils.select([".list-cell a"], document)
if(elements){
saveDetailContent(city, document);
Elements infoElements = JQueryUtils.select([".list-cell .li-info"], document)
Elements priceElements = JQueryUtils.select([".list-cell .li-side"], document)
if(infoElements){
// saveDetailContent(city, document);
saveLists(city, infoElements, priceElements)
}else{
flag = false;
}
@@ -289,6 +292,46 @@ class Esf58Crawl extends BaseBatch{
flag = false;
}
page++;
if(page>50){
flag = false;
}
}
}
def saveLists(city, infoElements, priceElements){
for(def i=0;i<infoElements.size();i++){
def e = infoElements.get(i)
def priceEle = priceElements.get(i)
def name = e.select(".li-title").text()
def avg_price = priceEle.select(".community-price strong").text()
def addressStr = e.select(".props span:eq(2)").text()
def region = null
def address = null
def href = e.select(".li-row").attr("href")
if(addressStr){
region = addressStr.toString().substring(0, addressStr.indexOf("-")).replace(" ","")
address = addressStr.toString().substring(addressStr.toString().lastIndexOf("-")+1, addressStr.toString().length())
}
if(avg_price){
avg_price = avg_price.toInteger()
}else{
avg_price = 0
}
def obj = findByParams(city, region, name);
if(obj){
DBObject udata2 = new BasicDBObject()
udata2.put("avg_price", avg_price);
udata2.put("c_date", new Date());
println "[update] "+city + " "+ name
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
}else{
println "[save] "+city + " "+ name
saveThirdParty(["url":href,"city":city,"region":region,"block":null,"name":name,"address": address, "avg_price":avg_price]);
}
}
}
@@ -310,18 +353,18 @@ class Esf58Crawl extends BaseBatch{
def detailDocument = CrawalUtil.getDocumentByABuYun(href, ERROR_MSG, CHARSET, "ee");
if(detailDocument){
def name = JQueryUtils.selectText(".title", detailDocument)
def navs = JQueryUtils.selectText(".breadcrumb-nav", detailDocument)
def navs = JQueryUtils.selectText(".crumbs-middle", detailDocument)
def info
if(navs){
info = navs.split(">")
}
def avg_price = JQueryUtils.selectText(".price", detailDocument);
def avg_price = JQueryUtils.selectText(".average", detailDocument);
def region;
def addr = JQueryUtils.selectText("td:contains(详细地址)+td", detailDocument);
def addr = JQueryUtils.selectText(".column-1:contains(小区地址) .value", detailDocument);
if(info){
if(info.size()>=4){
city = info[2].replace("小区","").trim()
region = info[3].replace("小区","").trim()
city = info[1].replace("小区","").trim()
region = info[2].replace("小区","").trim()
}
}
if(!addr){
@@ -17,7 +17,8 @@ class SynchDistrictPrice {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
def sourceMap = ["fang.com":"搜房","cityhouse.cn":"城市房产","5i5j.com":"我爱我家"]
def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多",
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()