This commit is contained in:
2021-09-08 16:15:13 +08:00
parent 1e73566aa4
commit b1f117c18f
8 changed files with 132 additions and 16 deletions
@@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
println city +" "+rows.size()
}
def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false);
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.dat",headers, "utf-8", false);
println "txt 生成"
}
@@ -49,6 +49,7 @@ class BigdataDealBatch extends BaseBatch{
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def statMap = getDistrictStatByCity(city, weekend)
@@ -445,6 +446,7 @@ class BigdataDealBatch extends BaseBatch{
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate));
q.put("area", new BasicDBObject('$gte', 20))
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
@@ -41,7 +41,7 @@ class BigdataDistrictBatch extends BaseBatch{
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.txt",headers, "utf-8",false);
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.dat",headers, "utf-8",false);
}
def createDatas(rows){
@@ -79,7 +79,7 @@ class CcbCommonUtils {
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
@@ -174,7 +174,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name", "美辰香醒")
// q.put("name", "石泉六村")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -30,7 +30,7 @@ class EvaluateRateBatch extends BaseBatch{
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false);
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.dat",headers, "utf-8", false);
}
def createDatas(def datas){
@@ -4,13 +4,20 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/**
* 建行模型估值项目
* 人工指导价同步
@@ -20,6 +27,8 @@ import java.util.concurrent.Executors
class GuiDingPriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
static main(args) {
runBatch(new GuiDingPriceBatch(), args);
@@ -34,6 +43,7 @@ class GuiDingPriceBatch extends BaseBatch{
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
if(prices){
def distMap = getDistrictMap(city)
@@ -42,18 +52,88 @@ class GuiDingPriceBatch extends BaseBatch{
println city + " " + rets.size()
}
//添加搜房的参考价
// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
// def datas = findFangDatas(districts)
// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
// rows.addAll(fangRets)
//添加保亭文昌的指导价
// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
// def cityDataMap = [:]
// for(def d : cityDatas){
// cityDataMap.put(d['A'], d['E'])
// }
// def personDatas = findPersonDatas(districts, cityDataMap);
// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
// rows.addAll(personRets)
}
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false);
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", true);
def rets = []
rets << headers
rets.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
// def rets = []
// rets << headers
// rets.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
}
def findFangDatas(def districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println " 小区量:"+districts.size()
def result = Collections.synchronizedList(new ArrayList());
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findMatchThirdPrice(dist)
if(obj && obj.avg_price){
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def findPersonDatas(def districts, def dataMap){
def result = []
for(def i=0;i<districts.size();i++) {
def dist = districts[i];
def price = dataMap.get(dist._id.toString())
if (price) {
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
println dist.city + " " + dist.region + " " + dist.name + " " + price
}
}
return result;
}
def findMatchThirdPrice(dist){
def q = new BasicDBObject()
q.put("match_city", dist.city)
q.put("match_region", dist.region)
q.put("match_name", dist.name)
q.put("s_date", sDate)
q.put("source","搜房")
def obj = dataProcessColl.findOne(q)
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
@@ -72,7 +152,7 @@ class GuiDingPriceBatch extends BaseBatch{
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << ps.guiding_person
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
@@ -85,8 +165,41 @@ class GuiDingPriceBatch extends BaseBatch{
return rets;
}
def createDatas(cityCodeDatas, distCodeMap, datas){
def rets = []
if(datas) {
for (def ps : datas) {
def ret = []
def areaData = distCodeMap.get(ps.region)
if (areaData) {
ret << ps.row_id
ret << ps.dist_id
ret << ps.name
ret << cityCodeDatas.get(ps.city)
ret << ps.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.avg_price
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{
results << headers
results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\staging_district.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\staging_district.txt",headers, "utf-8", false);
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING.dat",headers, "utf-8", false);
}
@@ -782,6 +782,7 @@ class CommonDatautils {
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("name_cat", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){