This commit is contained in:
2021-08-06 17:38:41 +08:00
parent 208d52949e
commit e423ab16cc
10 changed files with 64 additions and 58 deletions
@@ -34,7 +34,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "c_date", "m_date"])
def rows = createDatas(cityCodeDatas, dists)
def rows = createDatas(cityCodeDatas.get(city), dists)
rets.addAll(rows)
println city +" "+rows.size()
}
@@ -44,7 +44,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
}
def createDatas(cityCodeDatas, dists){
def createDatas(cityCode, dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -54,7 +54,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDatas, dist)
def ret = createData(cityCode, dist)
if(ret){
rets << ret
}
@@ -72,9 +72,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
return rets;
}
def createData(cityCodeDatas, dist){
def createData(cityCode, dist){
def ret = []
ret << cityCodeDatas.city_code
ret << cityCode
ret << dist._id.toString()
ret << CcbCodeUtils.parseBuildType(dist.build_type)
ret << ""
@@ -124,8 +124,6 @@ class BigdataDealBatch extends BaseBatch{
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}else{
println trade.city +" "+trade.fjRegion+" 没有找到对应的区域"
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
@@ -262,11 +260,11 @@ class BigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.fjRegion)
def data = distCodeMap.get(trade.region)
if(!data){
return ret
}else{
println trade.city +" "+trade.fjRegion+" 没有找到对应的区域"
println trade.city +" "+trade.region+" 没有找到对应的区域"
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
@@ -399,7 +397,7 @@ class BigdataDealBatch extends BaseBatch{
def startDate = null;
if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
}else{
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
}
@@ -415,7 +413,7 @@ class BigdataDealBatch extends BaseBatch{
if(trades){
tradesList.addAll(trades)
}
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+new Date().toLocaleString()
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -64,7 +64,7 @@ class BigdataForsaleBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = ["陵水"]//CcbCommonUtils.citys;
def index = 0
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
@@ -157,10 +157,10 @@ class BigdataForsaleBatch extends BaseBatch{
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
// def rets = []
// rets << headers
// rets.addAll(allDatas)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据")
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据")
}
@@ -12,7 +12,7 @@ import com.mongodb.DBObject
class CcbCommonUtils {
public static def citys = ["北京", "上海","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
public static def citys = ["上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -185,18 +185,18 @@ class CcbCommonUtils {
if(region){
q.put("region", region);
}
q.put("name", name);
q.put("cat", cat)
q.put("status", 0)
q.put("name_cat", 0);
q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
return obj;
for(def d : cursor){
if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
return d;
}
}
}
return null;
@@ -52,7 +52,7 @@ class EvaluateRateBatch extends BaseBatch{
}
private def findEvaluationRate(){
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = '小区'";
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'";
def rets = []
sqlInstance.eachRow(sql,{
rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated]
@@ -66,11 +66,11 @@ class ThirdPartyPriceBatch extends BaseBatch {
def data = distCodeMap.get(d.region)
if(data){
def ret = []
ret << third._id.toString()
ret << third.id.toString()
ret << d._id.toString()
ret << d.name
ret << cityCodeDatas.city_code
ret << cityCodeDatas.fangjia_city
ret << cityCodeDatas.get(d.city)
ret << d.city
ret << data.district_code
ret << data.district
ret << third.s_date
@@ -177,6 +177,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
dataMap.put("id", db._id.toString())
map.put(key, dataMap)
}
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){
@@ -240,7 +241,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def resultMap = [:]
map.each {key, value->
if(value){
if(value && value.size()>1){
resultMap.put(key, value)
}
}
@@ -15,7 +15,10 @@ class SeaweedDataMatchProcess {
if(!rMap.city){
def obj = CommonDatautils.findDistrict(city, region, name, "district")
if(!obj){
obj = CommonDatautils.findDistrictByAlias(city, region, name, "district")
def datas = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if(datas){
obj = datas[0]
}
}
if(obj){
rMap.estate_type2 = obj.estate_type2
@@ -40,14 +43,14 @@ class SeaweedDataMatchProcess {
if(rMap && rMap.id && region.contains(rMap.region)) {
def obj = CommonDatautils.findDistrictById(rMap.id.toString());
if (obj) {
rMap.estate_type2 = obj.estate_type2
}
}else{
rMap = null
}
// if(rMap && rMap.id && region.contains(rMap.region)) {
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
// if (obj) {
// rMap.estate_type2 = obj.estate_type2
// }
// }else{
// rMap = null
// }
if(!rMap && name && name.contains("·")){
def newName = trimChar(name).replace("·","")
rMap = matchFromDB2(city,newName)
@@ -30,14 +30,14 @@ class LocationTest {
// System.out.println(str);
def task = new LocationTest()
// task.run()
task.run()
// task.run2()
task.run3()
// task.run3()
// task.run4()
}
def run(){
def city = "南昌"
def city = "杭州"
def regionPointsMap = findRegionRange(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
def index = 0
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.price
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
@@ -11,32 +12,35 @@ class EvaluationBatch {
task.run()
}
def run(){
def filePath = "D:\\数据\\农行\\东莞农行测试数据(御景花园18条估值)(人工询价).xlsx"
def filePath = "D:\\数据\\银联\\房价网测试数据.xlsx"
def city = "东莞"
ExcelUtils.modify(filePath, { data ->
if(!data['M']){
// def address = data['C'].replace("\"","").trim()
// def rMap = SeaweedDataMatchProcess.doMatch(city, data['K'],data['L'], address );
def rMap = SeaweedDataMatchProcess.doMatch(city, data['D'],data['E'], null);
if(rMap){
def json = evaluation(rMap, data['H'], 7, data['I'],null)
def region = RegionData.formatRegion(data['A'], data['B'])
if(region){
def rMap = SeaweedDataMatchProcess.doMatch(data['A'], region, data['C'], null);
if(rMap){
def json = evaluation(rMap, data['D'], null, null,null)
// data['J'] = rMap.city
// data['K'] = rMap.region
// data['L'] = rMap.name
if(json){
def avgPrice = json.avg_price
def totalPrice = json.total_price
data['N'] = totalPrice
data['M'] = avgPrice
data['O'] = json.credit
data['P'] = rMap.region
data['Q'] = rMap.name
data['E'] = rMap.city
data['F'] = rMap.region
data['G'] = rMap.name
if(json){
def avgPrice = json.avg_price
def totalPrice = json.total_price
data['H'] = totalPrice
data['I'] = avgPrice
data['J'] = json.credit
// data['P'] = rMap.region
// data['Q'] = rMap.name
}
}
}
}
}, 0);
@@ -625,7 +625,7 @@ public class OpenApiClient {
params.put("token", TOKEN)
try {
JSONObject result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/evaluate", params)
if (result.getInt("code") == 200) {
if (result && result.getInt("code") == 200) {
if (result.containsKey("result")) {
// if (result.get("result").get("credit") == 0) {
// return null