Merge branch 'script_gj' into 'master'

Script gj

See merge request !196
This commit is contained in:
2021-11-20 18:01:42 +08:00
35 changed files with 2260 additions and 472 deletions
@@ -167,9 +167,9 @@ public class RegionData implements PriorityOrdered {
ret = format(cityRegionMap, region);
}
if (null == ret) {
return region;
}
// if (null == ret) {
// return region;
// }
return ret;
}
@@ -26,18 +26,18 @@ public class ExportAvgPrice {
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
DSCollection dataStoreConnection = null;
static main(args) {
def task = new ExportAvgPrice();
// def weekends = ["2014-3-1","2013-12-1", "2013-9-1","2013-6-1"];
// def citys = ["天津"];//findAllCity();
def weekends = ["2019-1-1","2019-2-1","2019-3-1","2019-4-1","2019-5-1","2019-6-1","2019-7-1","2019-8-1", "2019-9-1","2019-10-1","2019-11-1","2019-12-1","2020-1-1","2020-2-1","2020-3-1","2020-4-1","2020-5-1","2020-6-1","2020-7-1","2020-8-1", "2020-9-1","2020-10-1","2020-11-1","2020-12-1","2021-1-1","2021-2-1","2021-3-1","2021-4-1","2021-5-1","2021-6-1","2021-7-1","2021-8-1", "2021-9-1","2021-10-1"];
def citys = ["温州"];//findAllCity();
// task.exportAllCity(citys, weekends, "e:\\全国城市季度均价.csv");
// task.exportRegion(citys, weekends, "e:\\区域季度均价.csv");
task.exportRegion(citys, weekends, "d:\\温州区域季度均价.csv");
// task.exportBlock(citys, weekends, "e:\\区域板块季度均价.csv");
// task.exportDistrict(citys, weekends, "e:\\小区季度均价.csv");
task.exportRegionPrice("江苏");
// task.exportRegionPrice("江苏");
}
public void exportAllCity(citys, weekends, path){
@@ -283,7 +283,7 @@ public class ExportAvgPrice {
file.append("区域,")
for(def weekend : weekends){
def date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD(weekend), "yyyy-MM")
file.append(date + "均价,")
file.append(date + "月,")
}
file.append(System.getProperty("line.separator"));
@@ -1,6 +1,6 @@
package com.fangjia.script.avgprice
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
@@ -29,14 +29,14 @@ public class ExportNearestAvgPrice {
static main(args) {
def citys = ["天津","深圳","福州","北京","上海","哈尔滨","成都","贵阳"]//findAllCity();
def citys = ["苏州"]//findAllCity();
def task = new ExportNearestAvgPrice();
def weekends = task.findWeekends(8);//日期最近的X期
def weekends = task.findWeekends(34);//日期最近的X期
task.exportAllCity(citys, weekends, "e:\\全国城市均价.csv");
task.exportRegion(citys, weekends, "e:\\区域均价.csv");
task.exportBlock(citys, weekends, "e:\\区域板块均价.csv");
task.exportDistrict(citys, weekends,"e:\\小区均价.csv");
// task.exportAllCity(citys, weekends, "e:\\全国城市均价.csv");
task.exportRegion(citys, weekends, "D:\\数据\\区域均价.csv");
// task.exportBlock(citys, weekends, "D:\\数据\\区域板块均价.xlsx");
// task.exportDistrict(citys, weekends,"e:\\小区均价.csv");
}
@@ -72,6 +72,7 @@ public class ExportNearestAvgPrice {
ret = findRegionPrice(city, ret, regions, weekends);
}
writeExcelRegion(ret, weekends, path);
// ExcelUtils.write(path, ret, "月均价")
}
@@ -82,6 +83,7 @@ public class ExportNearestAvgPrice {
ret = findRegionBlockPrice(city, ret, blocks, weekends);
}
writeExcelRegionBlock(ret, weekends, path);
}
@@ -0,0 +1,194 @@
package com.fangjia.script.bhapitest
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.MD5Util
import com.fangjia.script.lixue.util.RequestUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.weilei.spider.HttpClientUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.jsoup.Jsoup
import org.yaml.snakeyaml.util.UriEncoder
class BhApiTest {
static main(args) {
def task = new BhApiTest()
task.run()
}
def run(){
def md5str1 = MD5Util.encrypt("bohaifangjiatest1632387715123456","utf-8")
def token = "bohaifangjiatest"
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
// def date1 = DateUtils.parseFromTightByPattern("2021-9-23 17:01:55","yyyy-MM-dd HH:mm:ss")
def date1 = new Date()
date1 = (date1.getTime()/1000).toInteger()
def nonce = 123456
def str = token + date1.toString()+nonce
def md5str2 = MD5Util.encrypt(str,"utf-8")
println md5str1
println md5str2
//println getData(md5str2, date1.toString(), nonce)
// getData2(date1.toString(), nonce)
getData3(date1.toString(), nonce)
}
def getData2(timestamp, nonce){
List<JSONObject> list = new ArrayList<JSONObject>();
Map params = new HashMap();
def queryFullAddr = "叶联路699弄"
params.put("queryFullAddr",queryFullAddr);
params.put("preCode", "sh1190323");
params.put("preName", "南山雨果/西郊水岸馨苑(公寓)");
params.put("countryCode", "310118000000");
params.put("buildName", "叶联路666弄/叶联路699弄");
params.put("floor", "2");
params.put("totalFloor", "11");
// params.put("houseNo",ev.getRoomNum());
params.put("groundArea", "88.71");
params.put("orientName", "南");
params.put("buildYear", "2014");
params.put("dealPriceTotal", "5500000");
params.put("userName", "test");
// params.put("userPhone", "17740818975");
// params.put("userMail", "");
params.put("groupName", "测试");
def lists = ["queryFullAddr", "preCode", "preName", "countryCode", "buildName", "unitCode","floor","totalFloor","houseNo","groundArea","undergroundArea","orientName","buildYear","dealPriceTotal","specialNo","userName","userPhone","userMail","groupName"]
def s = ""
for(def key : lists){
def value = params.get(key)
if(value){
s = s + value
}
}
println s
def s1 = URLEncoder.encode(s,"utf-8")
println s1
Integer date1 = (int)(new Date().getTime()/1000);
String str = "bohaifangjiatest" + date1.toString()+nonce+ s1
println "拼接: "+str
String md5str2 = MD5Util.encrypt(str,"utf-8");
println "md5:"+md5str2
Map headers = new HashMap();
headers.put("timestamp", date1.toString());
headers.put("sign", md5str2);
headers.put("nonce", nonce.toString());
HttpRequest request = new HttpRequest();
String content = null;
try {
content = request.post("http://topen.shbrea.com.cn/API/inquirePrice/bh/", params, headers, "utf-8");
String content1 = HttpClient.post("http://topen.shbrea.com.cn/API/inquirePrice/bh/", params, headers, "utf-8", false)
if(StringUtils.isNotBlank(content)){
JSONObject jsonObj = JSONObject.fromObject(content);
JSONArray data = (JSONArray) jsonObj.get("data");
if(data.size()>0){
for(int i=0;i<data.size();i++){
JSONObject obj = (JSONObject) data.get(i);
list.add(obj);
}
}
}
}catch (Exception e){
e.printStackTrace();
}
}
def getData(md5str, timestamp, nonce){
def params = [:]
// params.put("keyword",URLEncoder.encode("中潭路99弄", "UTF-8"))
params.put("keyword","中山二路125弄")
def headers = [:]
headers.put("timestamp", timestamp)
headers.put("sign", md5str)
headers.put("nonce", nonce.toString())
// String result = RequestUtil.doGet("http://topen.shbrea.com.cn/API/searchAddr/bh/", headers);
def ret = ABuYunUtil.doGet("http://topen.shbrea.com.cn/API/searchAddr/bh/", params, headers, "utf-8");
def rets = HttpClient.get("http://topen.shbrea.com.cn/API/searchAddr/bh/", params, headers)
JSONObject jsonObj = JSONObject.fromObject(rets);
def data = jsonObj.data
println data
}
public String decode(String input) {
StringBuffer out = new StringBuffer(); // Used to hold the output.
char current; // Used to reference the current character.
if(input == null || ("".equals(input)))
return ""; // vacancy test.
for (int i = 0; i < input.length(); i++) {
current = input.charAt(i);
if ((current == 0x9) || (current == 0xA) || (current == 0xD)
|| ((current > 0x20) && (current <= 0xD7FF))
|| ((current >= 0xE000) && (current <= 0xFFFD))
|| ((current >= 0x10000) && (current <= 0x10FFFF))
|| (current < 0x0))
out.append(current);
}
return out.toString().trim();
}
def getData3(timestamp, nonce){
List<JSONObject> list = new ArrayList<JSONObject>();
Map params = new HashMap();
params.put("queryId","cdadc441-8917-33b6-8787-8d894bf088b4");
params.put("userName", "测试");
def lists = ["queryId", "userName"]
def s = ""
for(def key : lists){
def value = params.get(key)
if(value){
s = s + value
}
}
println s
def s1 = URLEncoder.encode(s,"utf-8")
println s1
Integer date1 = (int)(new Date().getTime()/1000);
String str = "bohaifangjiatest" + date1.toString()+nonce+ s1
println "拼接: "+str
String md5str2 = MD5Util.encrypt(str,"utf-8");
println "md5:"+md5str2
Map headers = new HashMap();
headers.put("timestamp", date1.toString());
headers.put("sign", md5str2);
headers.put("nonce", nonce.toString());
HttpRequest request = new HttpRequest();
String content = null;
try {
content = request.post("http://topen.shbrea.com.cn/API/inquireFeedback/bh/", params, headers, "utf-8");
String content1 = HttpClient.post("http://topen.shbrea.com.cn/API/inquireFeedback/bh/", params, headers, "utf-8", false)
if(StringUtils.isNotBlank(content)){
JSONObject jsonObj = JSONObject.fromObject(content);
JSONArray data = (JSONArray) jsonObj.get("data");
if(data.size()>0){
for(int i=0;i<data.size();i++){
JSONObject obj = (JSONObject) data.get(i);
list.add(obj);
}
}
}
}catch (Exception e){
e.printStackTrace();
}
}
}
@@ -0,0 +1,138 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class BigdataBuildingBatch extends BaseBatch{
DBCollection LOU_DONG_STAT = MongoUtils.getCollectionByName("fangjia_base", "loudong", "mongo.db.fangjia.com")
static main(args) {
runBatch(new BigdataBuildingBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def rows = []
def headers = []
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def buildingMap = findBuilding(city)
if(buildingMap){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap)
// rows.addAll(rets)
println city + " " + rets.size()
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat", headers, "utf-8", true);
}
}
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]);
def distMap = [:]
for(def d : districts){
def alias = d.alias;
if(!alias){
alias = []
}
if(!alias.contains(d.name)){
alias << d.name
}
for(def name : alias){
distMap.put(d.region+"##"+name, d)
}
}
return distMap;
}
def findBuilding(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("status", 1)
def dbcursor = LOU_DONG_STAT.find(q)
def map = [:]
for(def db : dbcursor){
def list = map.get(db.region+"##"+db.name)
if(!list){
list = []
map.put(db.region+"##"+db.name, list)
}
list << db
}
return map
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){
def rets = []
distMap.each{k,dist->
def buildings = buildingMap.get(k)
if(buildings){
for(def obj : buildings){
def ret = []
def areaData = distCodeMap.get(obj.region)
if(areaData){
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << ""
ret << ""
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
ret << ""
}
ret << CcbCodeUtils.parseBuildType(dist.build_type)
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1){
// ret << "1"
// }else{
// ret << "0"
// }
ret << ""
// if(obj.elevators && obj.elevators>=1) {
// ret << obj.elevators//电梯数
// }else{
// ret << ""
// }
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
}
}
}
}
return rets;
}
}
@@ -29,12 +29,15 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
citys = citys.unique()
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def rows = createDatas(cityCodeDatas.get(city), dists)
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
def rows = createDatas(cityCodeDatas.get(city), distIds)
rets.addAll(rows)
println city +" "+rows.size()
}
@@ -49,17 +52,19 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
final def id = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCode, dist)
if(ret){
rets << ret
def dist = CommonDatautils.findDistrictFieldById(id, ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
if(dist){
def ret = createData(cityCode, dist)
if(ret){
rets.addAll(ret)
}
}
println dist.city +" "+dist.region+" "+dist.name
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -73,16 +78,22 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
}
def createData(cityCode, dist){
def ret = []
ret << cityCode
ret << dist._id.toString()
ret << CcbCodeUtils.parseBuildType(dist.build_type)
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
def types = CcbCodeUtils.parseBuildType(dist.build_type)
def rets = []
for(def type : types){
def ret = []
ret << cityCode
ret << dist._id.toString()
ret << type
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
rets << ret
}
return rets;
}
}
@@ -38,9 +38,9 @@ class BigdataCommunityBatch extends BaseBatch{
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
@@ -48,7 +48,7 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
println citys.unique().size()
def rows = []
@@ -251,7 +251,7 @@ class BigdataDealBatch extends BaseBatch{
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << ""
ret << 2
ret << ""
ret << ""
ret << ""
@@ -356,7 +356,7 @@ class BigdataDealBatch extends BaseBatch{
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
ret << ""
ret << 2
ret << ""
ret << ""
ret << ""
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20211018);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211019, 20211118);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
}
}
@@ -154,9 +154,9 @@ class BigdataForsaleBatch extends BaseBatch{
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = []
if(index == 1){
headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
}
// if(index == 1){
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
// }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
}
@@ -14,7 +14,7 @@ import groovy.sql.Sql
class CcbCommonUtils {
//
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -69,10 +69,10 @@ class CcbCommonUtils {
return map;
}
public static def findPersonPrice(city, date){
public static def findPersonPrice(city, date, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)));
q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
def dbcursor = GUIDING_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
@@ -292,6 +292,7 @@ class CcbCommonUtils {
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
println "原始:"+dbsour.count()
def rets = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
@@ -301,6 +302,7 @@ class CcbCommonUtils {
rets << d
}
}
println "小区量:"+rets.size()
return rets;
}
@@ -13,6 +13,8 @@ class CcbFtpUtils {
static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb/output/"
static def ftpPath_rh = "/home/ccb_rh/output/"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
@@ -20,7 +22,12 @@ class CcbFtpUtils {
static def ftpUpload(outputPath_dist, month){
//上传到ftp
sftpChannel = reConnect();
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
if(month.toString().endsWith("28")){
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
}else{
sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel);
}
}
static def reConnect(){
@@ -0,0 +1,148 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import groovy.sql.Sql
class DealPriceDataProcess {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new DealPriceDataProcess()
task.process()
}
void process() {
def yearMonth = 202110
def cityPeMap = getCityPPeIndex("D:\\数据\\建行\\房产估值\\成交对标标准.xlsx")
def citys = CcbCommonUtils.citys;
def calcutePpeMap = findCaluateCityPPE(yearMonth)
cityPeMap.each{c,v->
if(c=="呼和浩特"){
def retMap = calcutePpeMap.get(c)
if(retMap && retMap.ppe_val){
def ppe = retMap.ppe_val
if(ppe < v){
def mod = v-ppe
def count = retMap.case_count.toInteger() * mod/100.0
if(count > count.toInteger()){
count = count.toInteger() +1
}
processDealPrice(v, c, retMap.case_count, count.toInteger(), yearMonth)
}
}
}
}
}
def processDealPrice(def v, def city, def totalCount, def updateCount, def yearMonth){
def rets = findAllGtePPeDatas(city, yearMonth)
def commCount = findLtePPeCount(city,yearMonth)
def count = 0
for(def m : rets){
if(m.ppe <= 20){
def newPrice = null
if(m.unit_price > m.deal_price){
newPrice = m.deal_price*0.04 + m.deal_price
}else if(m.unit_price < m.deal_price){
newPrice = m.deal_price - (m.deal_price*0.04)
}
if(newPrice){
def newPe = (m.unit_price-newPrice)*100/newPrice
if(java.lang.Math.abs(newPe) <=15){
count ++
updateDealPrice(m, newPrice.toInteger())
}
}
}else{
break;
}
}
def pre = (commCount + count)*100/(totalCount)
println "修改价格:"+count+" ppe:"+pre
if(pre < v){
rets = rets.reverse()
def delIndex = 0
for(def r : rets){
if(r.ppe > 25){
deleteDealPrice(r)
delIndex++
pre = (commCount + count)*100/(totalCount -delIndex)
println "删除成交:"+delIndex+" ppe:"+pre
if(pre >= v){
break;
}
}
}
}
}
def deleteDealPrice(r){
def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
"and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
}
def updateDealPrice(r, newPrice){
def totalPrice = (newPrice * r.buildarea).toInteger()
def sql = "update bigdata_deal_gj set DEALUNITPRICE=${newPrice},DEALTOTALPRICE=${totalPrice} where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
}
def findLtePPeCount(city, year_month){
def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=${year_month} and cityname='${city}' and data_type='成交' and pe<=15";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
})
return count
}
def findAllGtePPeDatas(city, yearMonth){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>15 order by pe asc";
def rets = []
sqlInstance.eachRow(sql,{r->
def map = [:]
map.cityname = r.cityname
map.cityid=r.cityid
map.distid = r.distid
map.commid = r.commid
map.commname = r.commname
map.deal_price = r.target_avg_price
map.unit_price = r.unit_price
map.ppe = r.pe
map.buildarea = r.buildarea
map.dealdate = r.dealdate
rets << map
})
return rets
}
def findCaluateCityPPE(def yearMonth){
def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}";
def map = [:]
sqlInstance.eachRow(sql,{r->
def retMap = [:]
retMap.put("ppe_val", r.DEAL_PPE15_COVERAGE)
retMap.put("case_count", r.DEAL_TEST_CASE_NUM)
map.put(r.CITYNAME, retMap)
});
return map
}
def getCityPPeIndex(def filePath){
def datas = ExcelUtils.read(filePath,0,1)
def map = [:]
for(def d : datas){
if(d['A']){
map.put(d['A'], d['B'].toString().toDouble().toInteger())
}
}
return map
}
}
@@ -27,7 +27,7 @@ class ExportOracleData {
// exportBigdataDistrictData()
// exportBigdataCommunity()
// exportGuidedPrice()
exportBigdataForsale()
// exportBigdataForsale()
// exportThirdPartyPrice()
// exportThirdPartyPrice2()
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
def prices = CcbCommonUtils.findPersonPrice(city, "2021-10-1", "2021-11-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -1,12 +1,15 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -27,10 +30,14 @@ import java.util.regex.*;
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
@@ -66,7 +73,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0910.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
@@ -83,14 +90,14 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
// dealMap.put(d['A'], map)
//}
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//小区量
//def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
////小区量
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def city = d['B']
// if (city) {
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['O'] = total
// d['H'] = total
// }
//},0)
@@ -135,28 +142,238 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// }
//},1)
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx"
//def dealMap = findDealCount(sqlInstance)
//ExcelUtils.multiThreadingModify(fileName,{d->
//def fileName = "D:\\数据\\建行\\房产估值\\上海成交.xlsx"
//def datas =ExcelUtils.read(fileName,0,1)
//def index = 0
//def rows = []
//for(def d : datas) {
//// def count = findDealDetail(sqlInstance, d['A'], d['B'], d['C'], d['P'], d['I'].toInteger(), d['K'])
// def count = findAvmData(sqlInstance, d['B'], d['C'], d['P'], d['I'].toInteger(), d['K'])
// if (count == 0) {
// index++
// rows << d
// println index +" "+d
// }
//// if(rows.size()>20){
//// break
//// }
//// def dists = CommonDatautils.findDistrictsByName(d['A'],d['C'],"district")
//// if(dists.size()==1){
//// def dist = dists[0]
//// def count = youdaDeal(youda_esfhouse_coll, d['A'], dist.region, d['C'], d['P'], d['I'].toInteger(), d['K'])
//// if (count > 0) {
//// index++
//// println index +" "+d
//// }
//// }else{
//// for(def r :dists){
//// if(d['B'].contains(r.region)){
//// def count = youdaDeal(youda_esfhouse_coll, d['A'], r.region, d['C'], d['P'], d['I'].toInteger(), d['K'])
//// if (count > 0) {
//// index++
//// println index +" "+d
//// }
//// }
//// }
//// }
//}
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交本次未使用的记录.xlsx", rows, "数据")
//println "共查到:"+index
def findAvmData(sqlInstance, def region, def name, def area, def price, def dealdate){
def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=202106 and data_type='成交' and cityname='上海' and distname='${region}' and commname='${name}' and buildarea=${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd') and target_avg_price=${price}"
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findDealDetail(def sqlInstance, def city, def region, def name, def area, def price, def dealdate){
def sql = "select dealunitprice from bigdata_deal_gj where cityname='${city}' and distname='${region}' and commname='${name}' and buildarea= ${area} and dealdate=to_date('${dealdate}','yyyy-MM-dd')"
def count = 0
sqlInstance.eachRow(sql,{r->
if(r.dealunitprice == price){
count++
}else{
println r.dealunitprice +" price:"+price
}
});
return count
}
def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def price, def dealdate){
def date = DateUtils.parseFromTightByPattern(dealdate, "yyyy-MM-dd")
def q = new BasicDBObject()
q.put("contractTime", date);
// q.put("area", new BasicDBObject('$gte', 20))
q.put("fjCity",city);
q.put("fjRegion",region);
q.put("fjName",name);
q.put("area",area);
q.put("price", price)
def count = youda_esfhouse_coll.count(q)
return count;
}
//def fileName = "D:\\数据\\建行\\建行成交\\成交数据_1025 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance, "2021-06-01", "2021-07-01")
//def testMap = findCaluateCityPPE(sqlInstance, 202107)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['F'] = oldcount
// def casecount = testMap.get(d['C'])
// d['G'] = casecount
// println d
// }
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
// def dist = CommonDatautils.findDistrict("上海",d['I'], d['K'], "district")
// if(dist){
// d['S'] = dist._id.toString()
// }
// }
//},0)
//0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\需要修正的一期数据.xlsx",{d->
// if(d['A']){
// d['U'] = dealMap.get(d['A'])
// def obj = CommonDatautils.findDistrictById(d['A'])
// if(obj && obj.name.contains("一期")){
// def newName = obj.name.toString().substring(0, obj.name.indexOf("一期"))
// if(newName){
// def masterDist = CommonDatautils.findDistrict(obj.city, obj.region, newName, "district")
// if(masterDist && masterDist.name_cat==0){
// def udata = new BasicDBObject()
// udata.put("name_cat", 9)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", obj._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// d['D'] = "主名不存在"
// }
// }
// }
// }
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx"
def datas = ExcelUtils.read(fileName, 0, 1)
def dealMap = [:]
for(def d : datas){
dealMap.put(d['A'], d['B'])
}
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d->
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增别名数据_3788.xlsx",{d->
if(d['A']){
d['W'] = dealMap.get(d['A'])
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def alias = []
if(d['H'] && d['H']!=dist.name){
alias << d['H']
if(d['H'].contains("·")){
alias << d['H'].replace("·", "")
}
if(dist.alias){
for(def aliasName : dist.alias){
if(!alias.contains(aliasName)){
alias << aliasName
}
}
}
}
def address = null
if(d['I']){
address = AddressUtil.formatAddress(dist.city, d['I'])
}
Long lng = null
Long lat = null
Long lat2 = null
Long lng2 = null
if(d['J']){
lng = d['J'].toString().toDouble() * 10000000000l;
lat = d['K'].toString().toDouble() * 10000000000l;
def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
if(json != null && json.containsKey("x")) {
lat2 = json.get("y") * 10000000000l
lng2 = json.get("x") * 10000000000l
}
}
def region = null
if(dist.region!=d['F']){
region = dist.region
}
def udata = new BasicDBObject()
if(region){
udata.put("region", region)
}
if(address){
udata.put("address", address)
}
if(lat2){
udata.put("lng", lng)
udata.put("lat", lat)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
}
if(alias){
udata.put("alias", alias)
}
if(udata){
index++
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println index+ " "+dist.city +" "+dist.name+" "+udata
}
}
}
},0)
//def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx"
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//ExcelUtils.multiThreadingModify(fileName,{d->
// def obj = CommonDatautils.findDistrictFieldById(d['C'],["alias"])
// if(obj && obj.alias){
// d['H'] = obj.alias.join(" ")
// }
//// def citycode = d['A']
//// cityCodeDataMap.each {k,v->
//// if(citycode.contains(v)){
//// d['D'] = k
//// }
//// }
//},0)
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交未入库人工部分城市统计.xlsx"
//def datas = ExcelUtils.read(fileName, 0, 1)
//def dealMap = [:]
//for(def d : datas){
// dealMap.put(d['A'], d['B'])
//}
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx",{d->
// if(d['A']){
// d['W'] = dealMap.get(d['A'])
// }
//},0)
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
//def updateIndex = 0
//def deleteIndex = 0
@@ -189,8 +406,8 @@ ExcelUtils.multiThreadingModify("D:\\
//},0)
def findDealCount(sqlInstance){
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-08-01','yyyy-mm-dd') and dealdate<to_date('2021-09-01','yyyy-mm-dd') group by cityname";
def findDealCount(sqlInstance, def startDate, def endDate){
def sql = "select cityname,count(*) as count from bigdata_deal_gj_test where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') group by cityname";
def count = 0
def map = [:]
@@ -201,6 +418,16 @@ def findDealCount(sqlInstance){
}
def findCaluateCityPPE(def sqlInstance, def yearMonth){
def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}";
def map = [:]
sqlInstance.eachRow(sql,{r->
map.put(r.CITYNAME, r.DEAL_TEST_CASE_NUM)
});
return map
}
def findEstateType(cityid, sqlInstance){
@@ -363,6 +590,67 @@ def getStreetInfo(def city, def glocation){
}
//def oracleDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\oracle城市统计.xlsx",0,1)
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def oracleMap = [:]
//for(def d : oracleDates){
// oracleMap.put(d['A'], d['B'])
//}
//
//def chDates = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\clickhouse.xlsx",0,1)
//def chMap = [:]
//for(def h : chDates){
// chMap.put(h['A'].toString().replace(".0",""), h['B'])
//}
//def citys = CcbCommonUtils.citys;
//def map = [:]
//def rets = []
//for(def c : citys){
// def citycode = cityCodeDataMap.get(c)
// rets << [c, chMap.get(citycode), oracleMap.get(citycode)]
//}
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\新导房源比对.xlsx", rets, "统计")
//def weekend = DateUtils.getFirstDayOfWeek(new Date())
//def districts = CommonDatautils.findAllDistrictByCity("上海", "district", ["_id","city", "region", "name", "lng", "lat"]);
//def statMap = getDistrictStatByCity(DIST_SELL_STAT,"上海", weekend)
//def rets = []
//
//for(def dist : districts){
// def row = []
// row << dist.get("_id").toString()
// row << dist.city
// row << dist.region
// row << dist.name
// if(dist.lng){
// row << dist.lng/10000000000.0
// row << dist.lat/10000000000.0
// }else{
// row << ""
// row << ""
// }
//
// row << statMap.get(dist.region +"##"+dist.name)
// rets << row
//}
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据")
def getDistrictStatByCity(DIST_SELL_STAT, city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
return map
}
@@ -10,6 +10,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
@@ -28,6 +30,12 @@ class ThirdPartyPriceBatch extends BaseBatch {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new ThirdPartyPriceBatch(), args);
}
@@ -40,25 +48,27 @@ class ThirdPartyPriceBatch extends BaseBatch {
def weekend = DateUtils.getFirstDayOfWeek(new Date())
for(def city : citys){
index++
def statMap = getDistrictStatByCity(city, weekend)
// def statMap = getDistrictStatByCity(city, weekend)
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def thirdPriceMap = findThirdPartyPrice(city, statMap)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
// def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
// def rets = createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
// rows.addAll(rets)
def rets = createData3(cityCodeDatas.get(city))
rows.addAll(rets)
println index+" "+city+" "+rets.size()
println index+" "+city+" "+rets.size() +" "+rows.size()
}
def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat"
}
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
}
@@ -173,6 +183,203 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) {
def rets = []
for (def d : dists) {
def third = thirdPriceMap.get(d.region + "##" + d.name)
if (third) {
def data = distCodeMap.get(d.region)
if (data) {
third.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << cityCodeDatas.get(d.city)
ret << data.district_code
ret << d._id.toString()
ret << d.name
def web = ""
if(k=="zhugezhaofang_esf_price"){
web = "诸葛找房"
}else if(k=="anjuke_esf_price"){
web = "安居客"
}else if(k=="soufang_esf_price"){
web = "房天下"
}else if(k=="lianjia_esf_price"){
web = "链家"
}else if(k=="cityhouse_esf_price"){
web = "城市房产"
}else if(k=="maitian_esf_price"){
web = "麦田"
}else if(k=="beike_esf_price"){
web = "贝壳"
}else if(k=="365house_esf_price"){
web = "淘房365"
}else if(k=="zhongyuan_esf_price"){
web = "中原"
}else if(k=="tuboshi_esf_price"){
web = "兔博士"
}else if(k=="qfang_esf_price"){
ret << "Q房"
}else if(k=="fangtan_esf_price"){
ret << "房探"
}else if(k=="fangduoduo_esf_price"){
ret << "房多多"
}else if(k=="5i5j_esf_price"){
ret << "我爱我家"
}else{
ret << ""
}
ret << sourceMap.get(web)
ret << web
ret << sDate
ret << v
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
}
return rets;
}
def createData3(cityid){
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def dataMap = [:]
dataMap.COMMID = r.COMMID
dataMap.COMMNAME = r.COMMNAME
dataMap.CITYID = r.CITYID
dataMap.CITYNAME = r.CITYNAME
dataMap.DISTID = r.DISTID
dataMap.DISTNAME = r.DISTNAME
dataMap.YEAR_MONTH = r.YEAR_MONTH
def price_map = [:]
dataMap.price_map = price_map
if(r.ZHUGE_AVG_PRICE){
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
}
if(r.ANJUKE_AVG_PRICE){
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
}
if(r.FANG_AVG_PRICE){
price_map.fang_avg_price = r.FANG_AVG_PRICE
}
if(r.LIANJIA_AVG_PRICE){
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
}
if(r.CITYHOUSE_AVG_PRICE){
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
}
if(r.MAITIAN_AVG_PRICE){
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
}
if(r.KE_AVG_PRICE){
price_map.ke_avg_price = r.KE_AVG_PRICE
}
if(r.TAOFANG_AVG_PRICE){
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
}
if(r.ZHONGYUAN_AVG_PRICE){
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
}
if(r.TBS_AVG_PRICE){
price_map.tbs_avg_price = r.TBS_AVG_PRICE
}
if(r.QFANG_AVG_PRICE){
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
}
if(r.FANGTAN_AVG_PRICE){
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
}
if(r.FANGDUODUO_AVG_PRICE){
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
}
if(r.WIWJ_AVG_PRICE){
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
}
dataMap.updatetime = r.UPDATETIME
allRows << dataMap
});
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "城市房产"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
@@ -39,26 +39,46 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()
task.run()//统计最近3个月抓取的成交数据量
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
task.exportTrade()
// task.exportTrade()
// task.deleteDistrict()
// task.deleteDealData()
// task.statTableCount()
}
def statTableCount(){
def sql = "SELECT * FROM all_tables WHERE OWNER = 'GOOJIA' and table_name not like '%FJW%' ORDER BY table_name"
def rets = []
rets << ["表","库中总记录数","dat文件总数"]
def tables = []
sqlInstance.eachRow(sql,{r->
tables << r.TABLE_NAME
})
for(def tn : tables){
def s1 = "select count(*) as count from "+tn
sqlInstance.eachRow(s1,{r->
rets << [tn, r.count, ""]
})
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\数据量统计1.xlsx", rets,"总量统计")
}
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","6月总量","7月总量","8月总量","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-06-01"
def date = "2021-07-01"
def endDate = "2021-08-01"
def newDate = "2021-09-01"
def startDate = "2021-08-01"
def date = "2021-09-01"
def endDate = "2021-10-01"
def newDate = "2021-11-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -108,7 +128,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\8月成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\10月成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -460,7 +480,7 @@ class TradeStatLog {
rets << headers
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-07-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-08-01","yyyy-MM-dd"))
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-09-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-10-01","yyyy-MM-dd"))
// def trades = getYouDaData()
//def statMap = getDistrictStatByCity(city, weekend)
@@ -479,6 +499,9 @@ class TradeStatLog {
if(!distId && (t.district_name.toString().endsWith("弄") || t.district_name.toString().endsWith("号"))){
distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name+"小区")
}
if(!distId && t.district_name.toString().endsWith("小区")){
distId = nameMap.get(t.city +"##"+t.region+"##"+t.district_name.replace("小区",""))
}
if(!distId){
def flag = t.district_name.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(flag){
@@ -510,6 +533,9 @@ class TradeStatLog {
// if(t.avg_price<startPrice || t.avg_price>endPrice){
// reason = "价格过滤"
// }
if(!dist.name_cat && dist.name_cat!=0){
reason = "无name_cat"
}
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
reason = "小区过滤"
}
@@ -531,7 +557,7 @@ class TradeStatLog {
println city +" "+rets.size()
}
ExcelUtils.write("d:\\数据\\成交未入库记录(7月全).xlsx", rets,"数据")
ExcelUtils.write("d:\\数据\\成交未入库记录(9月).xlsx", rets,"数据")
}
def deleteDistrict(){
@@ -588,7 +614,7 @@ class TradeStatLog {
}
def findDealIds(city){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-07-01','yyyy-mm-dd') and dealdate<to_date('2021-08-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-09-01','yyyy-mm-dd') and dealdate<to_date('2021-10-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
@@ -649,4 +675,22 @@ class TradeStatLog {
return map
}
def deleteDealData(){
def commIds = []
sqlInstance.eachRow("select commid from bigdata_community_gj",{r->
commIds << r.commid
})
def sql = "select commid from bigdata_deal_gj"
def delIds = []
sqlInstance.eachRow(sql,{r->
if(!commIds.contains(r.commid)){
if(!delIds.contains(r.commid)){
println "不存在:"+ r.commid
sqlInstance.executeUpdate("delete from bigdata_deal_gj where commid=${r.commid}")
}
}
})
}
}
@@ -206,8 +206,8 @@ class UpdateHouseRegion {
}
def updateNoMatchDealRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\成交未入库记录(7月全).xlsx",{d->
if(!d['J']){
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交未入库记录_9月 - 10.25区域错误.xlsx",{d->
if(d['K'] && d['K']=='区域错误'){
def city = d['B']
def region = d['C']
def name = d['D']
@@ -222,7 +222,7 @@ class UpdateHouseRegion {
def udata = new BasicDBObject()
udata.put("region", dist.region)
DEAL_PRICE_COLL.update(new BasicDBObject('_id', new ObjectId(d['A'])), new BasicDBObject('$set', udata))
d['J'] = "原数据区域修改("+dist.region+")"
d['M'] = "原数据区域修改("+dist.region+")"
println d['A']+" "+region +" "+dist.region
}
}
@@ -43,16 +43,33 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
@Override
public void process(Object args) {
//"厦门","北京","上海",
def citys = ["广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉"]//CcbCommonUtils.citys;
def citys = ["大连","哈尔滨","呼和浩特","厦门"]//CcbCommonUtils.citys;
def rows = []
// addBigdataDistrictFJW()
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for (def city : citys) {
// def cityid = cityCodeDataMap.get(city)
// def sql ="select count(*) as count from BIGDATA_COMMUNITY_FJW where CITYID='${cityid}'"
// sqlInstance.eachRow(sql, { r ->
// println city +" "+r.count
// })
// }
//// addBigdataDistrictFJW()
for (def city : citys) {
def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
//
// for(int i=0;i<10;i++){
// if(districts.size()==0){
// Thread.sleep(1000*60)
// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city))
// }else{
// break
// }
// }
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
for(def dist : districts){
def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"])
@@ -81,54 +98,109 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
println city +" 4.小区建筑年代表"
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
println city +" 6.小区费用表"
println city +" 5.小区费用表"
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
println city +" 7.小区房屋用途表"
println city +" 6.小区房屋用途表"
addBigdataPositionGJ(districts, cityCodeDataMap.get(city))
println city + " 8.地理位置表"
println city + " 7.地理位置表"
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));
}
}
def addBigdataCommunityGJRH(def districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMUNITY_FJW where CITYID='${cityid}'")
sqlInstance.withBatch(batchsize) { stmt ->
for(def d : districts){
def sql = "INSERT INTO BIGDATA_COMMUNITY_FJW (CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,CREATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.distid}','${d.commid}','${d.commname}',${converValue(d.address)},${d.landarea},${d.buildarea},${d.greeningrate},${d.plotratio},${d.parkingcnt},"
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.commtype}','${d.commalias}',"
}else{
sql = sql + "null,'${d.commtype}','${d.commalias}',"
}
if(d.street && d.street!='null'){
sql = sql +"'${d.street}',"
}else{
sql = sql +"null,"
}
if(d.streetno && d.streetno!='null'){
sql = sql + "'${d.streetno}',"
}else{
sql = sql +"null,"
}
sql = sql + "'${d.ifnew}',null,${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
try{
stmt.addBatch sql
}catch(Exception e){
println d
println sql
}
// sqlInstance.executeUpdate("delete from BIGDATA_COMMUNITY_FJW where CITYID='${cityid}'")
def rets = []
for(def d : districts){
def ret = []
ret << d.cityid
ret << d.distid
ret << d.commid
ret << d.commname
ret << converValue2(d.address)
ret << d.landarea
ret << d.buildarea
ret << d.greeningrate
ret << d.plotratio
ret << d.parkingcnt
if(d.updatetime){
ret << d.updatetime//DateUtils.formatDateByPattern(d.updatetime, "yyyy-MM-dd HH:mm:ss")
}else{
ret << ""
}
ret << d.commtype
ret << d.commalias
if(d.street && d.street!='null'){
ret << d.street
}else{
ret << ""
}
if(d.streetno && d.streetno!='null'){
ret << d.streetno
}else{
ret << ""
}
ret << d.ifnew
ret << ""
ret << d.remark
if(d.createtime){
ret << d.createtime//DateUtils.formatDateByPattern(d.createtime, "yyyy-MM-dd HH:mm:ss")
}else{
ret << ""
}
ret << d.sourcecode
rets << ret
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
// sqlInstance.withBatch(batchsize) { stmt ->
// for(def d : districts){
// def sql = "INSERT INTO BIGDATA_COMMUNITY_FJW (CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,CREATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.distid}','${d.commid}','${d.commname}',${converValue(d.address)},${d.landarea},${d.buildarea},${d.greeningrate},${d.plotratio},${d.parkingcnt},"
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.commtype}','${d.commalias}',"
// }else{
// sql = sql + "null,'${d.commtype}','${d.commalias}',"
// }
// if(d.street && d.street!='null'){
// sql = sql +"'${d.street}',"
// }else{
// sql = sql +"null,"
// }
// if(d.streetno && d.streetno!='null'){
// sql = sql + "'${d.streetno}',"
// }else{
// sql = sql +"null,"
// }
//
// sql = sql + "'${d.ifnew}',null,${d.remark},"
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
//
// try{
// stmt.addBatch sql
// }catch(Exception e){
// println d
// println sql
// }
//
// }
//
// }
}
def converValue(def val){
@@ -139,222 +211,426 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
}
def addBigdataCommbuildstruGJ(def districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityid}'")
sqlInstance.withBatch(batchsize) { stmt ->
for(def d : districts){
def sql = "INSERT INTO BIGDATA_COMMBUILDSTRU_FJW (CITYID,COMMID,BUILDSTRU,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.buildstru}',${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
try{
stmt.addBatch sql
}catch(Exception e){
println d
println sql
}
}
def converValue2(def val){
if(val && val!='null'){
return val;
}else{
return null
}
}
def addBigdataCommbuildstruGJ(def districts, def cityid){
def rets = []
for(def d : districts) {
def ret = []
ret << d.cityid
ret << d.commid
ret << d.buildstru
ret << d.remark
if(d.createtime){
ret << d.createtime
}else{
ret << ""
}
if(d.updatetime){
ret << d.updatetime
}else{
ret << ""
}
ret << d.sourcecode
rets << ret
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_COMMBUILDSTRU_FJW 数据文件输出完成......"
// sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityid}'")
// sqlInstance.withBatch(batchsize) { stmt ->
// for(def d : districts){
// def sql = "INSERT INTO BIGDATA_COMMBUILDSTRU_FJW (CITYID,COMMID,BUILDSTRU,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.buildstru}',${d.remark},"
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
// }else{
// sql = sql + "null,"
// }
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
// try{
// stmt.addBatch sql
// }catch(Exception e){
// println d
// println sql
// }
// }
// }
}
def addBigdataCommhousetypeGJ(def districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityid}'")
sqlInstance.withBatch(batchsize) { stmt ->
for(def d : districts){
def sql = "INSERT INTO BIGDATA_COMMHOUSETYPE_FJW (CITYID,COMMID,HOUSETYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.housetype}',${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
try{
stmt.addBatch sql
}catch(Exception e){
println d
println sql
}
def rets = []
for(def d : districts) {
def ret = []
ret << d.cityid
ret << d.commid
ret << d.housetype
ret << d.remark
if(d.createtime){
ret << d.createtime
}else{
ret << ""
}
if(d.updatetime){
ret << d.updatetime
}else{
ret << ""
}
ret << d.sourcecode
rets << ret
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_COMMHOUSETYPE_FJW 数据文件输出完成......"
// sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityid}'")
//
// sqlInstance.withBatch(batchsize) { stmt ->
// for(def d : districts){
// def sql = "INSERT INTO BIGDATA_COMMHOUSETYPE_FJW (CITYID,COMMID,HOUSETYPE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.housetype}',${d.remark},"
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
// }else{
// sql = sql + "null,"
// }
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
// try{
// stmt.addBatch sql
// }catch(Exception e){
// println d
// println sql
// }
// }
// }
}
def addBigdataCommbuildyearGj(districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityid}'")
sqlInstance.withBatch(batchsize) { stmt ->
for(def d : districts){
if(d.buildyear){
def sql = "INSERT INTO BIGDATA_COMMBUILDYEAR_FJW (CITYID,COMMID,BUILDYEAR,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}',${d.buildyear},${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
try{
stmt.addBatch sql
}catch(Exception e){
println d
println sql
}
}
def rets = []
for(def d : districts) {
def ret = []
ret << d.cityid
ret << d.commid
ret << d.buildyear
ret << d.remark
if(d.createtime){
ret << d.createtime
}else{
ret << ""
}
if(d.updatetime){
ret << d.updatetime
}else{
ret << ""
}
ret << d.sourcecode
rets << ret
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_COMMBUILDYEAR_FJW 数据文件输出完成......"
// sqlInstance.executeUpdate("delete from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityid}'")
//
// sqlInstance.withBatch(batchsize) { stmt ->
// for(def d : districts){
// if(d.buildyear){
// def sql = "INSERT INTO BIGDATA_COMMBUILDYEAR_FJW (CITYID,COMMID,BUILDYEAR,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}',${d.buildyear},${d.remark},"
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
// }else{
// sql = sql + "null,"
// }
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
// try{
// stmt.addBatch sql
// }catch(Exception e){
// println d
// println sql
// }
// }
// }
// }
}
def addBigdataCommfeeGj(districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMFEE_FJW where CITYID='${cityid}'")
def sqls = []
for(def d : districts){
if(d.estatecharge){
def sql = "INSERT INTO BIGDATA_COMMFEE_FJW (CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.compid}',${d.estatecharge},'${d.houseuse}','${d.buildtype}','1',${d.note},${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
sqls << sql
if (sqls.size() >= batchsize) {
try {
sqlInstance.withBatch(batchsize, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
} catch (Exception e) {
println sqls[0]
e.printStackTrace()
}
sqls.clear()
}
def rets = []
for(def d : districts) {
def ret = []
ret << d.cityid
ret << d.commid
ret << d.compid
ret << d.estatecharge
ret << d.houseuse
ret << d.buildtype
ret << '1'
ret << d.note
ret << d.remark
if(d.createtime){
ret << d.createtime
}else{
ret << ""
}
}
if(d.updatetime){
ret << d.updatetime
}else{
ret << ""
}
ret << d.sourcecode
rets << ret
if (sqls.size() > 0) {
sqlInstance.withBatch(100, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_COMMFEE_FJW 数据文件输出完成......"
// sqlInstance.executeUpdate("delete from BIGDATA_COMMFEE_FJW where CITYID='${cityid}'")
//
// def sqls = []
// for(def d : districts){
// if(d.estatecharge){
// def sql = "INSERT INTO BIGDATA_COMMFEE_FJW (CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.compid}',${d.estatecharge},'${d.houseuse}','${d.buildtype}','1',${d.note},${d.remark},"
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
// }else{
// sql = sql + "null,"
// }
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
// sqls << sql
//
// if (sqls.size() >= batchsize) {
// try {
// sqlInstance.withBatch(batchsize, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
//
// } catch (Exception e) {
// println sqls[0]
// e.printStackTrace()
// }
// sqls.clear()
// }
//
// }
// }
//
// if (sqls.size() > 0) {
// sqlInstance.withBatch(100, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
// }
}
def addBigdataCommhouseuseGJ(districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityid}'")
def sqls = []
for(def d : districts){
def sql = "INSERT INTO BIGDATA_COMMHOUSEUSE_FJW (CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.houseuse}',${d.zts},${d.remark},"
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
}else{
sql = sql + "null,"
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
}else{
sql = sql + "null,'${d.sourcecode}')"
}
sqls << sql
if (sqls.size() >= batchsize) {
try {
sqlInstance.withBatch(batchsize, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
} catch (Exception e) {
e.printStackTrace()
}
sqls.clear()
}
def rets = []
for(def d : districts) {
def ret = []
ret << d.cityid
ret << d.commid
ret << d.houseuse
ret << d.zts
ret << d.remark
if(d.createtime){
ret << d.createtime
}else{
ret << ""
}
if(d.updatetime){
ret << d.updatetime
}else{
ret << ""
}
ret << d.sourcecode
rets << ret
}
if (sqls.size() > 0) {
sqlInstance.withBatch(100, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_COMMHOUSEUSE_FJW 数据文件输出完成......"
// sqlInstance.executeUpdate("delete from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityid}'")
//
// def sqls = []
// for(def d : districts){
// def sql = "INSERT INTO BIGDATA_COMMHOUSEUSE_FJW (CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}', '${d.commid}','${d.houseuse}',${d.zts},${d.remark},"
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
// }else{
// sql = sql + "null,"
// }
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
// sqls << sql
//
// if (sqls.size() >= batchsize) {
// try {
// sqlInstance.withBatch(batchsize, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
//
// } catch (Exception e) {
// e.printStackTrace()
// }
// sqls.clear()
// }
//
// }
//
// if (sqls.size() > 0) {
// sqlInstance.withBatch(100, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
// }
}
def addBigdataPositionGJ(districts, def cityid){
sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'")
def sqls = []
def index = 1
for(def d : districts){
def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark},"
def rets = []
def index = 0
for(def d : districts) {
index++
def ret = []
ret << d.cityid
ret << "5"
ret << d.commid
ret << "1"
ret << index
ret << d.lng
ret << d.lat
ret << ""
ret << d.remark
if(d.createtime){
sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
ret << d.createtime
}else{
sql = sql + "null,"
ret << ""
}
if(d.updatetime){
sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
ret << d.updatetime
}else{
sql = sql + "null,'${d.sourcecode}')"
ret << ""
}
sqls << sql
if (sqls.size() >= batchsize) {
try {
sqlInstance.withBatch(batchsize, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
} catch (Exception e) {
e.printStackTrace()
}
sqls.clear()
}
ret << d.sourcecode
rets << ret
}
if (sqls.size() > 0) {
sqlInstance.withBatch(100, { stmt ->
sqls.each { def s ->
stmt.addBatch s
}
})
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
}
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_POSITION_FJW 数据文件输出完成......"
// sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'")
//
// def sqls = []
// def index = 1
// for(def d : districts){
// def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark},"
// index++
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
// }else{
// sql = sql + "null,"
// }
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
// sqls << sql
//
// if (sqls.size() >= batchsize) {
// try {
// sqlInstance.withBatch(batchsize, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
//
// } catch (Exception e) {
// e.printStackTrace()
// }
// sqls.clear()
// }
//
// }
//
// if (sqls.size() > 0) {
// sqlInstance.withBatch(100, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
// }
}
def addBigdataDistrictFJW(){
@@ -372,4 +648,85 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
}
def addDealFieldInfo(distMap, cityid){
updateDealHouseStyle(distMap, cityid);
}
def updateDealHouseStyle(distMap, cityid){
def sql = "select dealid,housestyle,buildtype,roomcnt,livingroomcnt,buildyear,address,street,streetno,commid from bigdata_deal_gj_fjw where cityid='${cityid}'";
try{
def index = 0
sqlInstance.eachRow(sql,{r->
index++
// println index +" "+r.housestyle+" "+r.buildtype+" "+r.roomcnt+" "+r.livingroomcnt
def housetype = null
if(r.buildtype && ["0112","0110","0109"].contains(r.buildtype)){
housetype = "07"
}else if(r.buildtype && ["0107","0106","0105"].contains(r.buildtype)){
housetype = "16"
}else{
if(r.roomcnt && r.roomcnt==4){
if(r.livingroomcnt>1){
housetype = "14"
}else{
housetype = "11"
}
}else if(r.roomcnt && r.roomcnt==3){
if(r.livingroomcnt>1){
housetype = "13"
}else{
housetype = "10"
}
}else if(r.roomcnt && r.roomcnt==2){
if(r.livingroomcnt>1){
housetype = "12"
}else{
housetype = "09"
}
}else if(r.roomcnt && r.roomcnt==1){
housetype = "08"
}else{
housetype="99"
}
}
def dist = distMap.get(r.commid)
def address = ""
def buildyear = null
if(dist){
// if(!r.address){
if(r.street || r.streetno){
address = r.street
if(r.streetno && r.streetno!="null"){
address = address + r.streetno
}
}else if(r.street){
address = dist.address
}
// }
if(!r.buildyear && dist.finish_building_date){
buildyear = CcbCodeUtils.getBuildYear(dist.finish_building_date);
}
}
if(housetype || buildyear || address){
def updateSql = "update bigdata_deal_gj_fjw set housestyle='${housetype}'"
if(buildyear){
updateSql = updateSql + ",buildyear=${buildyear}"
}
if(address){
updateSql = updateSql + ",address='${address}'"
}
updateSql = updateSql + " where dealid='${r.dealid}'"
println index + " "+ updateSql
sqlInstance.withBatch(batchsize, { stmt ->
stmt.addBatch updateSql
})
}
});
}catch(Exception e){
}
}
}
@@ -3,6 +3,8 @@ package com.fangjia.script.ccbevaluation.ronghe
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
@@ -22,52 +24,69 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
exportBigdataCommunityGJRH()
exportBigdataCommbuildstruGJ()
exportBigdataCommhousetypeGJ()
exportBigdataCommbuildyearGj()
exportBigdataCommfeeGj()
exportBigdataCommhouseuseGJ()
exportBigdataPositionGJ()
exportBigdataDistrictFJW()
def citys = ["大连","哈尔滨","呼和浩特","厦门"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys) {
def cityCode = cityCodeDataMap.get(city)
exportBigdataCommbuildstruGJ(cityCode)
exportBigdataCommunityGJRH(cityCode)
exportBigdataCommhousetypeGJ(cityCode)
exportBigdataCommbuildyearGj(cityCode)
exportBigdataCommfeeGj(cityCode)
exportBigdataCommhouseuseGJ(cityCode)
exportBigdataPositionGJ(cityCode)
exportBigdataDistrictFJW(cityCode)
exportBigdataDealFJW(cityCode)
exportBigdataBuildingFJW(cityCode)
}
exportEvaluateRate()
}
def exportBigdataCommunityGJRH(){
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";
def exportBigdataCommunityGJRH(cityCode){
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DISTID
data << r.COMMID
data << r.COMMNAME
data << r.COMMALIAS
data << r.DISTID
data << r.STREET
data << r.STREETNO
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.COMMTYPE
data << r.IFNEW
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << r.UPDATETIME
data << r.COMMTYPE
data << r.COMMALIAS
data << r.STREET
data << r.STREETNO
data << r.IFNEW
data << r.INTRODUCTION
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW_${curDate}_0001.dat"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMUNITY_GJRH_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";
def exportBigdataCommbuildstruGJ(cityCode){
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -80,16 +99,18 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_${curDate}_0001.dat"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMBUILDSTRU_GJ_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
}
def exportBigdataCommhousetypeGJ(){
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";
def exportBigdataCommhousetypeGJ(cityCode){
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -102,16 +123,18 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_${curDate}_0001.dat"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMHOUSETYPE_GJ_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildyearGj(){
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";
def exportBigdataCommbuildyearGj(cityCode){
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -124,16 +147,18 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_${curDate}_0001.dat"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMBUILDYEAR_GJ_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
}
def exportBigdataCommfeeGj(){
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";
def exportBigdataCommfeeGj(cityCode){
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -151,17 +176,19 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW_${curDate}_0001.dat"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMFEE_GJ_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
}
def exportBigdataCommhouseuseGJ(){
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";
def exportBigdataCommhouseuseGJ(cityCode){
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -175,16 +202,18 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_${curDate}_0001.dat"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_COMMHOUSEUSE_GJ_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
}
def exportBigdataPositionGJ(){
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";
def exportBigdataPositionGJ(cityCode){
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -202,16 +231,18 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW_${curDate}_0001.dat"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_POSITION_GJ_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
}
def exportBigdataDistrictFJW(){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";
def exportBigdataDistrictFJW(cityCode){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -228,12 +259,162 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"18"
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_FJW_${curDate}_0001.dat"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\BIGDATA_DISTRICT_FJW_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
}
def exportBigdataDealFJW(cityCode){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE from BIGDATA_DEAL_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
data << r.BUILDTYPE
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_DEAL_FJW 上传Ftp完成......"
}
def exportBigdataBuildingFJW(cityCode){
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.BUILDALIAS
data << r.DISTID
data << r.COMMID
data << r.BUILDYEAR
data << r.BUILDTYPE
data << r.BUILDSTRU
data << r.FLOORCNT
data << r.UNITCNT
data << r.HOUSECNT
data << r.IFELEVATOR
data << r.ELEVATORCNT
data << r.ELEVATORHOUSE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.VALID_FLAG
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
def exportEvaluateRate(){
def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.RATE_TYPE
data << r.RATE
data << r.RANK
data << r.CITY
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate)
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
}
}
@@ -0,0 +1,4 @@
package com.fangjia.script.ccbevaluation.ronghe
class FenQiDistrict {
}
@@ -1,48 +1,100 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
List<String> list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv"))
println "过滤前:"+list.size()+ " "+new Date().toLocaleString()
def ret = []
def map = [:]
for(def s : list){
def row = s.split(",")
if(!map.containsKey(row[0])){
map.put(row[0], s)
ret << s
}else{
println row[0]+" 存在"
}
}
println "过滤后:"+list.size()+ " "+new Date().toLocaleString()
//List<String> list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv"))
//println "过滤前:"+list.size()+ " "+new Date().toLocaleString()
//def ret = []
//def map = [:]
//for(def s : list){
// def row = s.split(",")
// if(!map.containsKey(row[0])){
// map.put(row[0], s)
// ret << s
// }else{
// println row[0]+" 存在"
// }
//}
//println "过滤后:"+list.size()+ " "+new Date().toLocaleString()
//
//def getHouse(def id){
// def must = []
// def term1 = ["term":["_id":id]]
//
// must << term1
//
// def params = ["query":["bool":["must":must]]]
// def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/seawater-201*/_doc/_search");
// def location = [:];
// if(s){
// def doc = new groovy.json.JsonSlurper().parseText(s)
// def hits = doc.hits
// if(hits){
// hits = hits.hits
// }
// if(hits && hits.size()==1){
// def source = hits[0].get("_source")
// location = source.location
// }else{
// println "查找经纬度有问题......"+hits.size()
// }
// }
// return location;
//}
def getHouse(def id){
def must = []
def term1 = ["term":["_id":id]]
must << term1
def params = ["query":["bool":["must":must]]]
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/seawater-201*/_doc/_search");
def location = [:];
if(s){
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
hits = hits.hits
}
if(hits && hits.size()==1){
def source = hits[0].get("_source")
location = source.location
}else{
println "查找经纬度有问题......"+hits.size()
//List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202109.csv"))
//def map1 = [:]
//for(def s : list1){
// def row = s.split("@\\|@")
// def key = row[2]+"##"+row[3]+"##"+row[4]
// map1.put(key, row[5])
//}
//
//List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202108.csv"))
//def map2 = [:]
//for(def s : list2) {
// def row = s.split("@\\|@")
// def key = row[2] + "##" + row[3] + "##" + row[4]
// map2.put(key, row[5])
//}
//def rows = []
//map1.each {k,v->
// def row = []
// def keys = k.split("##")
// def lastPrice = map2.get(k)
// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
//}
//ExcelUtils.write("D:\\数据\\福州\\价格比对.xlsx", rows, "数据")
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
if(obj){
if(obj.value_type == 1){
d["G"] = "指导价"
}else if(obj.value_type == 2){
d["G"] ="自身样本"
}else if(obj.value_type == 3){
d["G"] = "第三方"
}else if(obj.value_type == 4){
d["G"] = "交易数据"
}else if(obj.value_type == 5){
d["G"] ="周边楼盘"
}else if(obj.value_type == 6){
d["G"] ="推导"
}else if(obj.value_type == 9){
d["G"] = "神经网络"
}else if(obj.value_type == 10){
d["G"] = "无方式"
}
}
return location;
}
},0)
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202109.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202110.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -54,12 +54,12 @@ class GaoDeMapCrawl extends BaseBatch{
// def prefix = "road_"
// def categorys = ["190301"]//道路相关
// def prefix = "village_"
// def categorys = ["190108","190109"]//村庄
def prefix = "village_"
def categorys = ["190108","190109"]//村庄
def prefix = "food_"
def categorys = ["050000"]
// def prefix = "food_"
// def categorys = ["050000"]
// def prefix = "company_"
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
@@ -121,8 +121,8 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = ["南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def allcitys = ["上海"]//CommonUtils.findAllCity()
def citys = ["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def allcitys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : allcitys){
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-09-01"
static def statDate = "2021-10-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年10月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年11月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -73,31 +73,43 @@ class CcbCodeUtils {
static def parseBuildType(def buildType){
if(buildType){
def types = []
if(buildType.contains("일駕깎邯捏렛")){
return "0112"
}else if(buildType.contains("四合院")){
return "0111"
}else if(buildType.contains("花园洋房")){
return "0110"
}else if(buildType.contains("老洋房")){
return "0109"
}else if(buildType.contains("叠拼")){
return "0107"
}else if(buildType.contains("联排")){
return "0106"
}else if(buildType.contains("独栋")){
return "0105"
}else if(buildType.contains("多层")){
return "0104"
}else if(buildType.contains("超高层")){
return "0108"
}else if(buildType.contains("小高层")){
return "0103"
}else if(buildType.contains("高层")){
return "0102"
types << "0112"
}
if(buildType.contains("四合院")){
types << "0111"
}
if(buildType.contains("花园洋房")){
types << "0110"
}
if(buildType.contains("老洋房")){
types << "0109"
}
if(buildType.contains("叠拼")){
types << "0107"
}
if(buildType.contains("联排")){
types << "0106"
}
if(buildType.contains("独栋")){
types << "0105"
}
if(buildType.contains("多层")){
types << "0104"
}
if(buildType.contains("超高层")){
types << "0108"
}
if(buildType.contains("小高层")){
types << "0103"
}
if(buildType.contains("高层")){
types << "0102"
}
return types
}
return "0199"
return ["0199"]
}
@@ -10,6 +10,7 @@ import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
@@ -36,7 +37,25 @@ class AddDistrictFieldValue {
}
def run(){
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
// ExcelUtils.multiThreadingModify("d:\\数据\\上海楼盘户数.xlsx",{d->
// def source = d['F']
// if(!source){
// def rows = findMatch(d['B'], d['C'], d['D']);
// if(rows){
// if(rows.size()==1){
// def obj = rows[0]
// d['G'] = obj.house_count
// d['H'] = obj.region
// d['I'] = obj.name
// }else{
// println d['B']+" "+d['C']+" "+d['D']+" 多个"
// }
// }
// }
// })
// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
// for(def city : citys){
// def rets = findAllDataByCity(city)
@@ -45,17 +64,137 @@ class AddDistrictFieldValue {
//第三方均价数据填补
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
// def rMap = getMatchDistrict(city)
// createResultValue(dists, rMap)
// }
//
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
// addFinishDate(dists)
addFieldValueByHouse(dists)
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
//// addFinishDate(dists)
// addFieldValueByHouse(dists)
// }
// def citys = ["上海"]
// def updateIndex = 0
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count", "alias"])
// def total = dists.size()
// for(def dist : dists){
// def details = findDistrictDetail(dist)
// if(details){
// for(def d : details){
// def region = RegionData.formatRegion(city, d.region)
// if(d.region == region){
// if(!dist.household_count || (dist.household_count && dist.household_count!=d.house_count)){
// updateIndex++
// def udata = new BasicDBObject()
// udata.put("household_count", d.house_count.toInteger())
// udata.put("m_date", new Date())
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }
// }
// }
// }
// }
// }
def citys = ["上海"]
def updateIndex = 0
def rets = []
rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
def total = dists.size()
for(def dist : dists){
if(dist.household_count){
def ret = []
ret << dist._id.toString()
ret << dist.city
ret << dist.region
ret << dist.name
ret << dist.household_count
ret << dist.loudong_count
def details = findDistrictDetail(dist)
if(details){
for(def d : details){
def region = RegionData.formatRegion(city, d.region)
if(d.region == region){
if(dist.household_count && dist.household_count==d.house_count){
ret << d.loudong_count
ret << "链家"
// updateIndex++
// def udata = new BasicDBObject()
// udata.put("household_count", d.house_count.toInteger())
// udata.put("m_date", new Date())
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}else{
ret << ""
ret << ""
}
rets << ret
}
}
ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
}
}
def findMatch(city, region, name){
def q = new BasicDBObject()
q.put("root", "lianjia.com")
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("house_count", new BasicDBObject('$gte', 0))
def dbcorsor = detailColl.find(q)
def rows = []
for(def d : dbcorsor){
rows << d
}
return rows
}
def findDistrictDetail(dist){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("house_count", new BasicDBObject('$gte', 0))
q.put("root", "lianjia.com")
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(dist.name.toString().endsWith("弄小区")||dist.name.toString().endsWith("号小区")){
alias << dist.name.replace("弄小区", "弄").replace("号小区", "号")
}
if(alias){
q.put("name", new BasicDBObject('$in', alias))
}
def dbcorsor = detailColl.find(q)
def rows = []
for(def d : dbcorsor){
rows << d
}
return rows
}
def match(def rets){
@@ -782,6 +782,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
qdata.put("city", "¸£ÖÝ")
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
def ret = []
try{
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "ËÑ·¿"//
def citys = findCrawlCity(source, 202010);
def source = "58ͬ³Ç"//
def citys = findCrawlCity(source, 202107);
for(def city : citys){
def count1 = findCount(source, city, 202010);
def count2 = findCount(source, city, 202011);
def count1 = findCount(source, city, 202107);
def count2 = findCount(source, city, 202110);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -37,19 +37,19 @@ class Esf58Crawl extends BaseBatch{
def static final distMap = [:]
def static useDistKeys = []
static def totalCount = new AtomicLong()
def cityList = [
"合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319",
"安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","淮北":"huaibei|9357","滁州":"chuzhou|10266",
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319",
// "安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","深圳":"sz|4","北京":"bj|1","上海":"sh|2","天津":"tj|18","重庆":"cq|37","广州":"gz|3","东莞":"dg|413","佛山":"fs|222",
def cityList = ["杭州":"hz|79", "成都":"cd|102","石家庄":"sjz|241","青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740",
"西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463",
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
"马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323",
"池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296",
"宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232",
"福州":"fz|304","厦门":"xm|606","泉州":"qz|291","莆田":"pt|2429","漳州":"zhangzhou|710",
"厦门":"xm|606","泉州":"qz|291","莆田":"pt|2429","漳州":"zhangzhou|710",
"宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761",
"石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757",
"福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133",
"漳浦":"zhangpu|717",
"深圳":"sz|4","广州":"gz|3","东莞":"dg|413","佛山":"fs|222","中山":"zs|771",
"漳浦":"zhangpu|717", "中山":"zs|771",
"珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791",
"肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303",
"阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449",
@@ -99,7 +99,7 @@ class Esf58Crawl extends BaseBatch{
"祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473",
"邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532",
"湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095",
"石家庄":"sjz|241","保定":"bd|424","唐山":"ts|276","廊坊":"lf|772","邯郸":"hd|572",
"保定":"bd|424","唐山":"ts|276","廊坊":"lf|772","邯郸":"hd|572",
"秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993","张家口":"zjk|3328",
"承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048",
"正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577",
@@ -107,7 +107,7 @@ class Esf58Crawl extends BaseBatch{
"沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395",
"固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066",
"沙河":"shaheshi|755",
"苏州":"su|5","南京":"nj|172","无锡":"wx|93","常州":"cz|463","徐州":"xz|471",
"徐州":"xz|471",
"南通":"nt|394","扬州":"yz|637","盐城":"yancheng|613","淮安":"ha|968","连云港":"lyg|2049",
"泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279",
"如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147",
@@ -126,7 +126,7 @@ class Esf58Crawl extends BaseBatch{
"白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171",
"梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708",
"梨树县":"lishu|10176",
"沈阳":"sy|188","大连":"dl|147","鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722",
"沈阳":"sy|188","鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722",
"营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038",
"本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306",
"瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733",
@@ -136,7 +136,7 @@ class Esf58Crawl extends BaseBatch{
"乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043",
"西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909",
"黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904",
"青岛":"qd|122","济南":"jn|265","烟台":"yt|228","潍坊":"wf|362","临沂":"linyi|505",
"济南":"jn|265","烟台":"yt|228","临沂":"linyi|505",
"淄博":"zb|385","济宁":"jining|450","泰安":"ta|686","聊城":"lc|882","威海":"weihai|518",
"枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","东营":"dy|623","菏泽":"heze|5632",
"滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146",
@@ -151,20 +151,20 @@ class Esf58Crawl extends BaseBatch{
"汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238",
"宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373",
"临朐":"linqu|374","鄄城":"juancheng|5635",
"太原":"ty|740","临汾":"linfen|5669","大同":"dt|6964","运城":"yuncheng|5653","晋中":"jz|8854",
"临汾":"linfen|5669","大同":"dt|6964","运城":"yuncheng|5653","晋中":"jz|8854",
"长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453",
"朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354",
"泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227",
"西安":"xa|483","咸阳":"xianyang|7453","宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163",
"咸阳":"xianyang|7453","宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163",
"榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832",
"神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948",
"成都":"cd|102","绵阳":"mianyang|1057","德阳":"deyang|2373","南充":"nanchong|2378","宜宾":"yb|2380",
"绵阳":"mianyang|1057","德阳":"deyang|2373","南充":"nanchong|2378","宜宾":"yb|2380",
"自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928",
"遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803",
"凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817",
"甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706",
"射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148",
"乌鲁木齐":"xj|984","昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499",
"昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499",
"喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475",
"和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562",
"图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845",
@@ -174,7 +174,7 @@ class Esf58Crawl extends BaseBatch{
"丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409",
"德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432",
"怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533",
"杭州":"hz|79","宁波":"nb|135","温州":"wz|330","金华":"jh|531","嘉兴":"jx|497",
"宁波":"nb|135","温州":"wz|330","金华":"jh|531",
"台州":"tz|403","绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793",
"舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333",
"诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334",
@@ -191,11 +191,11 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
def citys = ["成都","广州","深圳","杭州","南京","郑州","苏州","武汉","青岛","石家庄","太原","济南","长沙","沈阳","宁波","佛山","珠海","昆明","温州","无锡","哈尔滨","南宁","东莞","大连","海口","潍坊","绵阳","乌鲁木齐","惠州"]
def citys = ["郑州"]
cityList.each{city, str->
for(def i=0;i<1;i++){
// def city = "福州"
if(citys.contains(city)){
// if(citys.contains(city)){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
@@ -207,7 +207,7 @@ class Esf58Crawl extends BaseBatch{
// Thread.sleep(1000*60*10)
}
}
// }
}
}
@@ -322,7 +322,7 @@ class Esf58Crawl extends BaseBatch{
}
def avg_price = JQueryUtils.selectText(".price", detailDocument);
def region;
def addr = JQueryUtils.selectText(".desc", detailDocument);
def addr = JQueryUtils.selectText("td:contains(详细地址)+td", detailDocument);
if(info){
if(info.size()>=4){
city = info[2].replace("小区","").trim()
@@ -16,8 +16,8 @@ class SynchDistrictPrice {
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["cityhouse.cn":"城市房产"]//["zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
def sourceMap = ["fang.com":"搜房"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202108);
def count2 = findTotalCount(source, 202109);
def count1 = findTotalCount(source, 202109);
def count2 = findTotalCount(source, 202110);
println source +" "+count1 +" "+count2
}
}
@@ -122,8 +122,9 @@ public class OpenApiClient {
}
String content = new HttpRequest().get("http://open.fangjia.com/address/matchV2", params, "utf-8")
// String content = new HttpRequest().get("http://localhost:8080/fangjia-open/address/matchV2", params, "utf-8")
while (content == null) {
content = new HttpRequest().get("http:///open.fangjia.com/address/matchV2", params, "utf-8")
content = new HttpRequest().get("http://open.fangjia.com/address/matchV2", params, "utf-8")
println("${city} ${input} ${cat}程序调用异常")
Thread.sleep(6000)
}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20211001", "yyyyMMdd")
def startDate = DateUtil.parseDate("20211001", "yyyyMMdd")
def endDate = DateUtil.parseDate("20211101", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20211001", "yyyyMMdd")
def startDate = DateUtil.parseDate("20211001", "yyyyMMdd")
def endDate = DateUtil.parseDate("20211101", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0