抓无锡村庄
This commit is contained in:
@@ -258,8 +258,9 @@ public class BaiDuApiUtils {
|
||||
params.put("coords", lng.toString() + "," + lat.toString());
|
||||
params.put("from", String.valueOf(from));
|
||||
HttpRequest request = new HttpRequest();
|
||||
String result = null;
|
||||
try{
|
||||
String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
|
||||
result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(result);
|
||||
while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){
|
||||
changeAK();
|
||||
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 小区建筑类型数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommbuildtypeBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"])
|
||||
def rows = createDatas(cityCodeDatas, distCodeMap, dists)
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
}
|
||||
|
||||
def createDatas(cityCodeDatas, distCodeMap, dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCodeDatas, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
|
||||
println dist.city +" "+dist.region+" "+dist.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(cityCodeDatas, distCodeMap, dist){
|
||||
def ret = []
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << dist._id.toString()
|
||||
ret << CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
+69
-59
@@ -49,7 +49,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
rows.addAll(rets)
|
||||
println city +" "+rets.size()
|
||||
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def allDatas = []
|
||||
@@ -61,16 +61,20 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){
|
||||
def rets = []
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
def dist = dists[i]
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets << createData(cityCodeDatas, distCodeMap, dist)
|
||||
def ret = createData(cityCodeDatas, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -85,70 +89,76 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
def createData(def cityCodeDatas, def distCodeMap, def dist){
|
||||
def ret = []
|
||||
ret << cityCodeDatas.city_code
|
||||
if(dist.region){
|
||||
ret << distCodeMap.get(dist.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
try{
|
||||
ret << cityCodeDatas.city_code
|
||||
if(dist.region){
|
||||
ret << distCodeMap.get(dist.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist._id.toString()
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << dist._id.toString()
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.all_area
|
||||
ret << dist.build_area
|
||||
ret << dist.virescence_percent
|
||||
ret << dist.cubage_percent
|
||||
ret << dist.car_site_count
|
||||
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
|
||||
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
|
||||
ret << dist.household_count
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "物业小区"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
|
||||
ret << number
|
||||
ret << dist.all_area
|
||||
ret << dist.build_area
|
||||
ret << dist.virescence_percent
|
||||
ret << dist.cubage_percent
|
||||
ret << dist.car_site_count
|
||||
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
|
||||
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
|
||||
ret << dist.household_count
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "物业小区"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
|
||||
ret << number
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << 0
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
}catch(Exception e){
|
||||
println dist.city + " "+dist.name
|
||||
e.printStackTrace()
|
||||
}
|
||||
ret << 0
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 成交数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataDealBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
for(def city : citys){
|
||||
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
// def trades = getCityData(city)
|
||||
// def rets = createData(trades, cityCodeDatas, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
|
||||
def youDataTrade = getYouDaData()
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
|
||||
println city +" "+youdaDatas.size()
|
||||
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rets)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(rows)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def trade = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
if(source == "youda"){
|
||||
result << converYouDaData(trade, cityCodeDatas, distCodeMap)
|
||||
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
}else{
|
||||
result << converData(trade, cityCodeDatas, distCodeMap)
|
||||
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def obj = CommonDatautils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
|
||||
if(!obj){
|
||||
return ret;
|
||||
}
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << trade._id.toString()
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
if(trade.fjRegion){
|
||||
ret << distCodeMap.get(trade.fjRegion)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.region
|
||||
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.endsWith("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
ret << CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
ret << cityCodeDatas.city_code
|
||||
ret << trade._id.toString()
|
||||
ret << cityCodeDatas.fangjia_city
|
||||
if(trade.region){
|
||||
ret << distCodeMap.get(trade.region)
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.region
|
||||
if(trade.address){
|
||||
if(trade.address.toString().endsWith("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
ret << number
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.dist_id.toString()
|
||||
ret << trade.master_name
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
|
||||
ret << ""//期数
|
||||
ret << trade.floor
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(trade.room, trade.hall, trade.toilet)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.avg_price
|
||||
ret << trade.total_price
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
def root = sourceMap.get(trade.source)
|
||||
if(root){
|
||||
if(CcbCommonUtils.rootMap.get(root)){
|
||||
ret << CcbCommonUtils.rootMap.get(root)
|
||||
}else{
|
||||
ret << "99"
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.total_price
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getHouseRoom(def room, def hall, def toilet){
|
||||
def s = ""
|
||||
if(room){
|
||||
s = s + room +"室"
|
||||
}
|
||||
if(hall){
|
||||
s = s + hall +"厅"
|
||||
}
|
||||
if(toilet){
|
||||
s = s + toilet +"卫"
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
def getCityData(def city){
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"])
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, DateUtils.getFirstDayOfMonth(new Date()))
|
||||
if(trades){
|
||||
tradesList.addAll(trades)
|
||||
}
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return tradesList
|
||||
}
|
||||
|
||||
def getYouDaData(){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate));
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
for(def obj : dbcursor){
|
||||
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房"].contains(obj.houseType)){
|
||||
rets << obj;
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
+5
-1
@@ -34,13 +34,14 @@ class BigdataDistrictBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def allDatas = createDatas(rows)
|
||||
|
||||
def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"]
|
||||
|
||||
def rets = []
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8");
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8",false);
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
@@ -76,6 +77,9 @@ class BigdataDistrictBatch extends BaseBatch{
|
||||
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def lists = []
|
||||
for(def db : dbcursor){
|
||||
if(db.fangjia_district && db.fangjia_district=="二道"){
|
||||
println db.fangjia_district+" "+city
|
||||
}
|
||||
if(db.district_code){
|
||||
def m = [:]
|
||||
m.id = db._id.toString()
|
||||
|
||||
+6
-5
@@ -54,7 +54,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataForsaleBatch(), args);
|
||||
@@ -125,10 +125,11 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
latch.await()
|
||||
println city + " 房源总量:"+retMap.size()
|
||||
def allDatas = createSellDatas(city, retMap.values());
|
||||
// def allDatas = createSellDatas(city, retMap.values());
|
||||
def allDatas = retMap.values()
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true);
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt",headers, "utf-8", true);
|
||||
println "txt文件生成 "
|
||||
|
||||
}
|
||||
@@ -340,8 +341,8 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
if(rootMap.get(seawater.root)){
|
||||
rowMap.root = rootMap.get(seawater.root)
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
-521
@@ -1,521 +0,0 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 出售房源数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataForsaleBatch2 extends BaseBatch{
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataForsaleBatch2(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
for(def city : citys){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def distMap = getDistrictMap(city)
|
||||
exportSellCSV(city, cityCodeDatas, distCodeMap, distMap, 20200701, 20210731);
|
||||
}
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
if(d.alias){
|
||||
d.alias.each{name->
|
||||
if(!distMap.containsKey(d.region+"##"+name)){
|
||||
distMap.put(d.region+"##"+name, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
def exportSellCSV(city, cityCodeDatas, distCodeMap,distMap, startDate, endDate){
|
||||
println city +" 小区key:"+distMap.size()
|
||||
def cityCode = cityCodeDatas.city_code
|
||||
def retMap = findHouses(city, distMap, "sell", startDate,endDate, cityCode, distCodeMap)
|
||||
def allDatas = createSellDatas(city, retMap);
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale_${city}.txt",headers, "utf-8", true);
|
||||
println "txt文件生成 "
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createSellDatas(def city, def map){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(map.size())
|
||||
println city +"总房源量:"+map.size()
|
||||
|
||||
def datas = []
|
||||
map.each{k,m->
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
datas << converData(m)
|
||||
println totalCount.get() +" "+city+" "+m.region+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return datas ;
|
||||
}
|
||||
|
||||
def converData(def m){
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
data << m.region
|
||||
data << m.district_id
|
||||
data << m.district_name
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.address==null?"":m.address==null
|
||||
data << m.build_year==null?"":m.build_year
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << ""
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
data << m.hall==null?"":m.hall
|
||||
data << ""
|
||||
data << m.toilet==null?"":m.toilet
|
||||
data << m.direction_code==null?"":m.direction_code
|
||||
data << m.fitment_code==null?"":m.fitment_code
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.p_date==null?"":m.p_date
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.root
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.c_date==null?"":m.c_date
|
||||
data << m.m_date==null?"":m.m_date
|
||||
data << m.source
|
||||
data << ""
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def distMap, def cat, def startDate, def endDate, def cityCode, def distCodeMap){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
def retMap = new ConcurrentHashMap()
|
||||
println city +" 没过滤前:"+ houses.size()
|
||||
if(houses){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(houses.size())
|
||||
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
final def map = houses[k];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def rowMap = converData(distMap, map, cat)
|
||||
if(retMap){
|
||||
retMap.put(rowMap.houseId, rowMap)
|
||||
println totalCount.get() +" [匹配success]"+city+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
}else{
|
||||
println totalCount.get() +" [匹配fail]"+city+" "+m.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
}
|
||||
latch.await()
|
||||
return retMap;
|
||||
}
|
||||
|
||||
def converData(distMap, map, cat){
|
||||
def rMap = distMap.get(map.region +"##"+map.district_name)
|
||||
if(!rMap){
|
||||
return null
|
||||
}
|
||||
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(!seawater.total_price){
|
||||
return null
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
return null
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
if(seawater.height && seawater.height<1){
|
||||
return null
|
||||
}
|
||||
if(seawater.height && seawater.height>60){
|
||||
return null
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=5 || seawater.room<1){
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
return null
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
return null
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
if(rootMap.get(seawater.root)){
|
||||
rowMap.root = rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
|
||||
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
def regionCode = distCodeMap.get(rMap.region)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
rowMap.city_code = cityCode
|
||||
rowMap.region = rMap.region
|
||||
rowMap.region_code = regionCode
|
||||
rowMap.district_id = rMap.get("_id").toString()
|
||||
rowMap.district_name = rMap.name
|
||||
rowMap.build_year = buildYear;
|
||||
rowMap.proptype = housePro
|
||||
rowMap.propyears = propertyYear;
|
||||
rowMap.estate_type = estateType
|
||||
rowMap.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
rowMap.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
rowMap.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
rowMap.address = rMap.address
|
||||
rowMap.build_structure_code = struCode
|
||||
rowMap.house_structure_code = null//getHouseStructureCode();
|
||||
rowMap.source = "FJW"
|
||||
return rowMap
|
||||
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
|
||||
}
|
||||
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
|
||||
return pdate;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String csvHandlerStr(String str) {
|
||||
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
|
||||
String tempDescription=str;
|
||||
//如果有逗号
|
||||
if(str.contains(",")||str.contains(",")){
|
||||
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
|
||||
if(str.contains("\"")){
|
||||
// tempDescription=str.replace("\"", "\"\"");
|
||||
tempDescription=str.replace("\"", "");
|
||||
}
|
||||
//在将逗号转义
|
||||
tempDescription="\""+tempDescription+"\"";
|
||||
}
|
||||
return tempDescription;
|
||||
}
|
||||
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 生成一个用不重复的ID
|
||||
*/
|
||||
def GenNonDuplicateID(randomLength){
|
||||
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def marge(){
|
||||
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
|
||||
def ids = new HashSet();
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
index++
|
||||
if(index>1){
|
||||
def arrs = it.split("\\|@\\|")
|
||||
def id = arrs[1]
|
||||
if(!ids.contains(id)){
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
}
|
||||
+49
-2
@@ -7,8 +7,8 @@ import com.mongodb.DBCollection
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","长春","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
|
||||
// public static def citys = ["北京"]
|
||||
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
|
||||
// public static def citys = ["武汉"]
|
||||
|
||||
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
|
||||
|
||||
@@ -18,6 +18,11 @@ class CcbCommonUtils {
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
|
||||
|
||||
public static def findCityCodeValue(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fangjia_city", city)
|
||||
@@ -58,4 +63,46 @@ class CcbCommonUtils {
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
public static def findTradesDetail(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
d.address = dist.address
|
||||
d.master_name = dist.name
|
||||
d.build_type = dist.build_type
|
||||
d.build_structure = dist.build_structure
|
||||
d.house_roperty = dist.house_roperty
|
||||
d.property_life = dist.property_life
|
||||
d.estate_type2 = dist.estate_type2
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class CityBatch extends BaseBatch{
|
||||
rets << headers
|
||||
rets.addAll(allDatas)
|
||||
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.xlsx", rets, "数据")
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8");
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_city.dat",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def createDatas(rows){
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
import groovy.sql.Sql
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
for(def city : citys) {
|
||||
def data = CcbCommonUtils.findCityCodeValue(city)
|
||||
def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'";
|
||||
|
||||
def count = 0
|
||||
sqlInstance.execute(sql,{
|
||||
count = it.count
|
||||
});
|
||||
println city +" "+count
|
||||
}
|
||||
|
||||
|
||||
+31
-28
@@ -74,20 +74,20 @@ class AddThirdPriceDistrict {
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceDistrict();
|
||||
// task.run()
|
||||
task.run()
|
||||
|
||||
def mapName = "锦华星光苑3,4幢"
|
||||
// def mapName = "锦华星光苑3,4幢"
|
||||
//// println task.pattern("港新花苑A1区");
|
||||
println mapName.replaceAll("\\w+-?\\w+(幢)", "")
|
||||
println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
|
||||
// println mapName.replaceAll("\\w+-?\\w+(幢)", "")
|
||||
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
|
||||
}
|
||||
def run(){
|
||||
// exportDatas();
|
||||
exportDatas();
|
||||
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
// addInfo();
|
||||
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
|
||||
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
|
||||
addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
|
||||
}
|
||||
|
||||
def addCrawlMapInfo( def path){
|
||||
@@ -135,29 +135,32 @@ class AddThirdPriceDistrict {
|
||||
|
||||
|
||||
def exportDatas(){
|
||||
def city = "无锡"
|
||||
def datas = findMatchErrorDatas(city);
|
||||
println "共【"+datas.size()+"】条"
|
||||
datas = filterName(datas);
|
||||
println "过滤后共【"+datas.size()+"】条"
|
||||
def resMap = groupByName(datas);
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
|
||||
resMap.each {k,v->
|
||||
def ret = []
|
||||
def arrs = k.split("##")
|
||||
ret << arrs[0]
|
||||
ret << arrs[1]
|
||||
if(arrs.size()==3 && arrs[2]){
|
||||
ret << arrs[2]
|
||||
}else{
|
||||
ret << ""
|
||||
//跑完 "上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞",
|
||||
def citys = ["佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
for(def city : citys){
|
||||
def datas = findMatchErrorDatas(city);
|
||||
println "共【"+datas.size()+"】条"
|
||||
datas = filterName(datas);
|
||||
println "过滤后共【"+datas.size()+"】条"
|
||||
def resMap = groupByName(datas);
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","优选地址","来源网站","高德区域","高德名称","高德地址","高德类型","经度","纬度","百度区域","百度名称","百度地址","百度类型","经度","纬度","标记","匹配区域","匹配名称","小区ID","地址"]
|
||||
resMap.each {k,v->
|
||||
def ret = []
|
||||
def arrs = k.split("##")
|
||||
ret << arrs[0]
|
||||
ret << arrs[1]
|
||||
if(arrs.size()==3 && arrs[2]){
|
||||
ret << arrs[2]
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << resAddrMap.get(k)
|
||||
ret << v.join(",")
|
||||
rets << ret
|
||||
}
|
||||
ret << resAddrMap.get(k)
|
||||
ret << v.join(",")
|
||||
rets << ret
|
||||
ExcelUtils.write("D:\\数据\\小区\\"+city+"第三方均价.xlsx", rets,"最后")
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\"+city+"第三方均价.xlsx", rets,"最后")
|
||||
}
|
||||
|
||||
def filterName(datas){
|
||||
@@ -465,7 +468,7 @@ class AddThirdPriceDistrict {
|
||||
def findMatchErrorDatas(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("s_date",202005)
|
||||
q.put("s_date",new BasicDBObject('$gte', 202101))
|
||||
// q.put("name","博济聚创科技园")
|
||||
// q.put("source","贝壳找房")
|
||||
def dbcursor = matchErrorLogColl.find(q)
|
||||
|
||||
@@ -56,11 +56,11 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
// def prefix = "road_"
|
||||
// def categorys = ["190301"]//道路相关
|
||||
|
||||
// def prefix = "village_"
|
||||
// def categorys = ["190108","190109"]//村庄
|
||||
def prefix = "village_"
|
||||
def categorys = ["190108","190109"]//村庄
|
||||
|
||||
def prefix = "company_"
|
||||
def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
|
||||
// def prefix = "company_"
|
||||
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
@@ -118,7 +118,8 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys =["无锡"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
|
||||
def districts = ["广州_增城_保利山水大院", "广州_天河_保利天汇", "广州_增城_保利平方花园", "广州_天河_睿山", "广州_天河_博雅御轩", "广州_白云_保利紫云", "广州_花都_保利花园海德","广州_荔湾_保利花漾花园", "广州_南沙_保利半岛", "广州_南沙_保利保宏花园", "广州_南沙_保利星海小镇", "广州_荔湾_葛洲坝保利曼城", "广州_荔湾_保利葛洲坝海德公馆", "广州_越秀_淘金中行宿舍", "广州_白云_保利紫云B区"]
|
||||
|
||||
def headers = ["输入","城市","区域","小区名","小区别名","地址","地址别名","均价(元/㎡ )","经度","纬度","板块","环线","物业类型","建筑类型","商圈","户数","物业费,例如:12.0元/平米/月","停车位","停车位(描述)","产权年限","产权性质","开工时间","开盘时间","竣工时间","交房时间","占地面积","建筑面积","容积率","绿化率","预售许可证","开发商","物业公司","装修情况","建筑结构","周边配套","交通状况","相关信息","描述","售楼地址","售楼电话","折扣信息","楼栋总数","建成年代"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
|
||||
for(def s : districts){
|
||||
def arrs = s.split("_")
|
||||
def m = OpenApiClient.districtDetail(arrs[0], arrs[1], arrs[2])
|
||||
def ret = []
|
||||
ret << s
|
||||
if(m){
|
||||
ret << m.city
|
||||
ret << m.district
|
||||
ret << m.name
|
||||
if(m.alias){
|
||||
ret << m.alias.toString().replace("[","").replace("]","")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << m.address
|
||||
if(m.addressAlias){
|
||||
ret << m.addressAlias.toString().replace("[","").replace("]","")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << m.avgPrice
|
||||
ret << m.lng
|
||||
ret << m.lat
|
||||
ret << m.block
|
||||
ret << m.loopLine
|
||||
ret << m.propertyType
|
||||
ret << m.buildingType
|
||||
ret << m.commercialArea
|
||||
ret << m.totalUnitCount
|
||||
ret << m.managementFee
|
||||
ret << m.parking
|
||||
ret << m.parkingDescription
|
||||
ret << m.ownershipPeriod
|
||||
ret << m.ownershipType
|
||||
ret << m.startBuildTime
|
||||
ret << m.startSellingTime
|
||||
ret << m.finishBuildTime
|
||||
ret << m.moveinTime
|
||||
ret << m.floorSpace
|
||||
ret << m.constructionSpace
|
||||
ret << m.plotRatio
|
||||
ret << m.greenPercentage
|
||||
ret << m.preSalePermit
|
||||
ret << m.developer
|
||||
ret << m.propertyManagement
|
||||
ret << m.decorationStandard
|
||||
ret << m.buildingStructure
|
||||
ret << m.supportingFacilities
|
||||
ret << m.transportation
|
||||
ret << m.relatedInfo
|
||||
ret << m.description
|
||||
ret << m.salesOfficeAdd
|
||||
ret << m.salesPhoneNumber
|
||||
ret << m.discount
|
||||
ret << m.buildingCount
|
||||
ret << m.finishYear
|
||||
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\亲邻\\楼盘详情数据.xlsx", rets, "数据")
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user