Merge remote-tracking branch 'origin/master'

This commit is contained in:
2023-02-20 19:03:01 +08:00
40 changed files with 3674 additions and 182 deletions
@@ -24,6 +24,7 @@ import com.fangjia.open.model.*;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.Term;
@@ -85,7 +86,6 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService;
import com.fangjia.service.open.WaitCrawlKeywordsService;
import com.fangjia.service.sql.FjtEvaluationService;
import com.fangjia.service.web.AddressService;
import org.sothis.web.mvc.annotation.Param;
@Authority(role = Role.USER)
public class AddressController {
@@ -116,6 +116,8 @@ public class AddressController {
private static final String GRADE_CONSUME = "消费评分";
public final static OpenConfig CONFIG = OpenConfig.getConfig();
private static final String ES_USER = "developer";
private static final String ES_PASSWORD = "5e8edd851d2fdfbd7415232c67367cc3";
/**
* @deprecated see parse2Action
@@ -521,8 +523,65 @@ public class AddressController {
}
MatchDistrictResult matchResult = addressMatchProxyService.checkMatchDistrictCorrect(param);
jsonResult.put("matchResult", matchResult);
if (jsonResult.containsKey("category")) {
if ("office".equals(jsonResult.getString("category"))) {
jsonResult.put("propertyType", "写字楼");
} else if ("shop".equals(jsonResult.getString("category"))) {
jsonResult.put("propertyType", "商铺");
}
}
} else if (jsonResult != null) {
jsonResult.put("matchResult", new MatchDistrictResult());
// 判断是否为自然村
boolean matchedVillage = false;
try {
if (jsonResult.get("searchAddress") != null) {
JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress");
String city = searchAddress.containsKey("city") ? searchAddress.getString("city") : null;
String region = searchAddress.containsKey("district") ? searchAddress.getString("district") : null;
String name = searchAddress.containsKey("keywords") ? searchAddress.getString("keywords") : null;
String township = searchAddress.containsKey("township") ? searchAddress.getString("township") : null;
if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) {
List<JSONObject> villages = findVillage(city, region, name);
if (CollectionUtils.isNotEmpty(villages)) {
if (villages.size() == 1 || StringUtils.isBlank(township)) {
JSONObject village = villages.get(0).getJSONObject("_source");
jsonResult.put("id", villages.get(0).getString("_id"));
jsonResult.put("province", village.getString("province"));
jsonResult.put("city", village.getString("city"));
jsonResult.put("district", village.getString("region"));
jsonResult.put("township", village.getString("township"));
jsonResult.put("name", village.getString("name"));
jsonResult.put("lng", village.getJSONObject("location").getDouble("lon"));
jsonResult.put("lat", village.getJSONObject("location").getDouble("lat"));
jsonResult.put("propertyType", "自然村");
matchedVillage = true;
} else {
for (JSONObject source : villages) {
JSONObject village = source.getJSONObject("_source");
if (village.getString("township").contains(township) || township.contains(village.getString("township"))) {
jsonResult.put("id", source.getString("_id"));
jsonResult.put("province", village.getString("province"));
jsonResult.put("city", village.getString("city"));
jsonResult.put("district", village.getString("region"));
jsonResult.put("township", village.getString("township"));
jsonResult.put("name", village.getString("name"));
jsonResult.put("lng", village.getJSONObject("location").getDouble("lon"));
jsonResult.put("lat", village.getJSONObject("location").getDouble("lat"));
jsonResult.put("propertyType", "自然村");
matchedVillage = true;
break;
}
}
}
}
}
}
} catch(Exception ex) {
ex.printStackTrace();
}
if (matchedVillage == false) {
jsonResult.put("matchResult", new MatchDistrictResult());
}
}
resultModel.setResult(json.get("result"));
mav.setModel(resultModel);
@@ -918,6 +977,60 @@ public class AddressController {
return ret;
}
private JSONObject addTerm(String name, String value){
JSONObject term = new JSONObject();
JSONObject match = new JSONObject();
match.put(name, value);
term.put("term", match);
return term;
}
/***
* 获取自然村
* @param city
* @param region
* @param name
* @return
*/
private List<JSONObject> findVillage(String city, String region, String name) {
JSONObject reqBody = new JSONObject();
reqBody.put("from", 0);
reqBody.put("size", 5);
JSONObject query = new JSONObject();
JSONObject bool = new JSONObject();
JSONArray must = new JSONArray();
must.add(addTerm("city", city));
must.add(addTerm("region", region));
must.add(addTerm("name", name));
bool.put("must", must);
query.put("bool", bool);
reqBody.put("query", query);
Base64 token = new Base64();
String authenticationEncoding = token.encodeAsString((""+ES_USER+":"+ES_PASSWORD+"").getBytes());
HttpRequest request = new HttpRequest();
Map<String, String> headers = new HashMap<String, String>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Basic "+authenticationEncoding+"");
try {
String response = request.postWithBody("http://192.168.0.45:9200/village/_search", reqBody, headers, "UTF-8");
JSONObject json = JSONObject.fromObject(response);
if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("hits")){
return Collections.emptyList();
}
JSONArray hits = json.getJSONObject("hits").getJSONArray("hits");
List<JSONObject> results = new ArrayList<JSONObject>();
for(int i=0 ;i < hits.size(); i++){
JSONObject jsonObject = hits.getJSONObject(i);
results.add(jsonObject);
}
return results;
} catch (HttpRequestException e) {
e.printStackTrace();
}
return Collections.emptyList();
}
/**
* 加载测试数据
*
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -22,6 +23,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataCommbuildtypeBatch(), args);
}
@@ -29,14 +33,14 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
citys = citys.unique()
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
// def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city));
def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
def rows = createDatas(cityCodeDatas.get(city), distIds)
rets.addAll(rows)
println city +" "+rows.size()
@@ -50,7 +54,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"]
FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false);
println "txt 生成"
println "txt 生成->导Oracle"
saveToPgSql(rets)
}
@@ -103,4 +109,35 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
}
return rets;
}
def saveToPgSql(rows){
def sqls = []
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
def sql = "insert into base.t0831_bigdata_commbuildtype_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
println "start: "+new Date()
def index =0
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
index++
println index
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
}
}
println "end: "+new Date()
}
}
@@ -12,6 +12,7 @@ import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -30,6 +31,9 @@ class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
}
@@ -37,7 +41,6 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -71,7 +74,9 @@ class BigdataCommunityBatch extends BaseBatch{
allDatas << headers
allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
println "txt 生成->导Oracle"
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
@@ -169,7 +174,7 @@ class BigdataCommunityBatch extends BaseBatch{
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger()
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "pa101"
@@ -222,4 +227,34 @@ class BigdataCommunityBatch extends BaseBatch{
}
return ret;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -50,6 +50,9 @@ class BigdataDealBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataDealBatch(), args);
}
@@ -69,43 +72,45 @@ class BigdataDealBatch extends BaseBatch{
addDealPriceMatch(city);
// }
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
saveToPgSql(rows)
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -680,4 +685,35 @@ class BigdataDealBatch extends BaseBatch{
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
}
}
println "end: "+new Date()
}
}
@@ -21,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -57,15 +58,22 @@ class BigdataForsaleBatch extends BaseBatch{
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
}
@Override
public void process(Object args) {
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -74,7 +82,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
@@ -102,6 +110,33 @@ class BigdataForsaleBatch extends BaseBatch{
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate)
println city + "(原)房源总量:"+retMap.size()
def allDatas = retMap.values()
allDatas = repeatHouseId(allDatas)
println city + "(新)房源总量:"+allDatas.size()
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
}
// def allDatas = createSellDatas(city, retMap.values());
def headers = []
// if(index == 1){
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
// }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成->手动导入到Oracle "
saveToPgSql(allDatas)
}
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
@@ -116,7 +151,8 @@ class BigdataForsaleBatch extends BaseBatch{
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
@@ -166,21 +202,38 @@ class BigdataForsaleBatch extends BaseBatch{
})
}
latch.await()
return retMap
}
def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat"
def repeatHouseId(allDatas){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(allDatas.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<allDatas.size();i++){
final def houseInfo = allDatas[i];
fixedThreadPool2.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = isExistFromHouse(houseInfo[1])
if(count){
println houseInfo[1]+" 已存在 "+totalCount.get()
}else{
rets << houseInfo
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println city + " 房源总量:"+retMap.size()
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = []
// if(index == 1){
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
// }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
latch.await()
return rets
}
@@ -427,11 +480,25 @@ class BigdataForsaleBatch extends BaseBatch{
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
// if(rets1){
// def newRets = []
// def cityCode = cityCodeDataMap.get(city)
// def houseIds = getAllHouseIds(commId, cityCode)
// for(def r : rets1){
// if(!houseIds.contains(r.houseId)){
// newRets << r
// }else{
// println r.houseId +" 已存在"
// }
// }
// return newRets
// }
// return [];
return rets1;
}
@@ -466,8 +533,11 @@ class BigdataForsaleBatch extends BaseBatch{
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
@@ -608,6 +678,20 @@ class BigdataForsaleBatch extends BaseBatch{
}
rets << rowMap
// if(houseIds.contains(seawater.id)){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
// def count = isExistFromHouse(seawater.id)
// if(count){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
}
}
@@ -653,9 +737,11 @@ class BigdataForsaleBatch extends BaseBatch{
queryBuilder.must(inBuilder)
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
@@ -804,6 +890,19 @@ class BigdataForsaleBatch extends BaseBatch{
}
rets << rowMap
// if(houseIds.contains(seawater.id)){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
// def count = isExistFromHouse(seawater.id)
// if(count){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
}
}
@@ -959,12 +1058,6 @@ class BigdataForsaleBatch extends BaseBatch{
}
/*
* 生成一个用不重复的ID
*/
def GenNonDuplicateID(randomLength){
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
}
@@ -987,14 +1080,56 @@ class BigdataForsaleBatch extends BaseBatch{
println ids.size()
}
def getDistrictIds(def cityid){
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
def getAllHouseIds(def commid, def cityCode){
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'";
def distIds = new HashSet()
def houseIds = new HashSet()
sqlInstance.eachRow(sql,{it->
distIds << it.commid
houseIds << it.FORSALEID
});
return distIds;
return houseIds;
}
def isExistFromHouse(def houseId){
def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";
def count = null
sqlInstance.eachRow(sql,{it->
count = it.count
});
return count;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_forsale_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40],
r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -16,7 +16,8 @@ import groovy.sql.Sql
class CcbCommonUtils {
//
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
@@ -60,7 +61,7 @@ class CcbCommonUtils {
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="252"){
citys = CcbCommonUtils.citys_252
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
}else{
@@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils
class CcbFtpTestUtils {
static def ftpServer = "192.168.0.86"//45
static def ftpServer = "192.168.0.45"//45
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
@@ -25,9 +25,9 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()
task.deleteThirdPartyData()
// task.delDealByPe()
task.delDealData()
// task.delDealData()
}
def delDealByPe(){
@@ -372,12 +372,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202211
def month = 202212
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202212)
def rets = findPeGteThirdParty(city, 202301)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -13,6 +13,7 @@ import org.apache.commons.codec.digest.DigestUtils
import java.sql.ResultSet
//估值 新版 导出所有数据
class ExportOracleAllDataTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
@@ -20,7 +21,7 @@ class ExportOracleAllDataTest {
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def citys = null
def curDate = "20221228"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def curDate = "20220128"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
static main(args) {
def task = new ExportOracleAllDataTest()
@@ -38,7 +39,7 @@ class ExportOracleAllDataTest {
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)//["潍坊","济宁","威海","临沂","绵阳","陇南","塔城","阿勒泰","迪庆州","日喀则"] //CcbCommonUtils.getRunningCitys(args)
def curMonth = "202212"//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
@@ -31,7 +31,7 @@ class ExportOracleEvaluationData {
exportCityPrice(curMonth)
exportRegionPrice(curMonth)
exportCommunityBasePriceGj(curMonth)//已合下面的合并
// exportCommunityBasePriceGj(curMonth)//已合下面的合并
exportCommunityBasePrice(curMonth)
exportCommunityPdescribeGj(curMonth)
@@ -248,6 +248,7 @@ class ExportOracleEvaluationData {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -11,6 +11,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
@@ -32,6 +33,10 @@ class GuiDingPriceBatch extends BaseBatch{
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new GuiDingPriceBatch(), args);
}
@@ -77,10 +82,7 @@ class GuiDingPriceBatch extends BaseBatch{
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
// def rets = []
// rets << headers
// rets.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据")
saveToPgSql(rows)
}
@@ -211,5 +213,34 @@ class GuiDingPriceBatch extends BaseBatch{
return distMap;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(300, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20])
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,20 @@
package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
class PgsqlCommUtils {
public static def database = "warehouse"
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/"+database+"?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
public static def findAllDistrictIdsByOracle(def cityid){
def sql = "select gujia_comm_id from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def set = new HashSet()
sqlInstance.eachRow(sql,{r->
set << r.gujia_comm_id
});
return set
}
}
@@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.bson.types.ObjectId
/**
@@ -18,6 +19,9 @@ import org.bson.types.ObjectId
@BatchOwner(Watcher.GaoJuan)
class StagingDistrictBatch extends BaseBatch{
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new StagingDistrictBatch(), args);
}
@@ -47,6 +51,8 @@ class StagingDistrictBatch extends BaseBatch{
//ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
saveToPgSql(rows)
}
def createCityData(def dists, def fenQiMap){
@@ -86,4 +92,34 @@ class StagingDistrictBatch extends BaseBatch{
}
return fenQiMap;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -45,8 +45,91 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\小区地址名单_估家科技V2 (3).xlsx",{d->
if(d['J']){
def cityId = cityCodeDataMap.get(d['H'])
def map = findCcbPriceCredit(sqlInstance236, cityId, d['J'])
println d['H'] +" "+d['J']+ map.toString()
if(map){
d['Q'] = map.credit
d['R'] = map.month
d['U'] = map.price
}
// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J'])
// if(thirdMap){
// d['S'] = thirdMap.price
// d['T'] = thirdMap.month
// }
}
})
def findCcbPriceCredit(sqlInstance, cityid, commid){
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.month = r.year_month
map.credit = r.CONFIDENCE
map.price = r.avg_price
});
return map
}
def findThirdPriceBySource(sqlInstance, cityid, commid){
def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" +
"order by price_date desc) WHERE ROWNUM <= 1"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.month = r.price_date
map.price = r.community_price
});
return map
}
for(def c : citys){
def cityCode = cityCodeDataMap.get(c)
println c +" "+cityCode
def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301)
//
checkNoPriceDist(sqlInstance236, priceDists, cityCode)
}
def checkNoPriceDist(sqlInstance236,priceDists, cityid){
def commids = []
def sql = "select commid from bigdata_community_gj where cityid='${cityid}' and status=0";
sqlInstance236.eachRow(sql, { r ->
commids << r.commid
})
for(def commid : commids){
if(!priceDists.contains(commid)){
updateDistStatus(sqlInstance236, commid)
}
}
return commids
}
def updateDistStatus(sqlInstance236, commid){
def sql = "update bigdata_community_gj set status=1 where commid='${commid}'";
sqlInstance236.executeUpdate(sql)
}
def findAllPriceDist(sqlInstance236, cityid, yearMonth){
def commids = []
def sql = "select commid from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} ";
sqlInstance236.eachRow(sql, { r ->
commids << r.commid
})
return commids
}
//def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
//
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -1087,15 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){
//}
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['C'] = dist.city
d['D'] = dist.region
d['E'] = dist.name
}
})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['C'] = dist.city
// d['D'] = dist.region
// d['E'] = dist.name
// }
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx"
//def datas = ExcelUtils.read(fileName, 0, 1)
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202212//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202210
def endDate = 202212
def sDate= 202301//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202211
def endDate = 202301
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -36,6 +36,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new ThirdPartyPriceBatch(), args);
}
@@ -69,6 +73,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
saveToPgSql(rows)
}
@@ -618,4 +624,32 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18])
}
}
println "end: "+new Date()
}
}
@@ -40,7 +40,7 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
@@ -92,13 +92,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-09-01"
def date = "2022-10-01"
def endDate = "2022-11-01"
def newDate = "2022-12-01"
def startDate = "2022-10-01"
def date = "2022-11-01"
def endDate = "2022-12-01"
def newDate = "2023-01-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -0,0 +1,115 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
class TestData {
static main(args) {
def task = new TestData()
// task.run()
task.update()
}
def update(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\ÁúÑÒ1.xlsx", { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
if(d['E'] && d['G']=="È˹¤"){
def q = new BasicDBObject()
def baidu = BaiDuApiUtils.geoconv(d['E'].toString().toDouble(), d['F'].toString().toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
q.put("lat2", dLat2)
q.put("lng2", dLng2)
Long lng = d['E'].toString().toDouble() * 10000000000L
Long lat = d['F'].toString().toDouble() * 10000000000L
q.put("lng", lng)
q.put("lat", lat)
DbUtils.seaweed.update(new BasicDBObject('_id',d['A']),new BasicDBObject('$set',q))
}
}
});
}
def run(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\ÁúÑÒ.xlsx", { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def loc = getLocation(dist.city, dist.region, dist.name)
if(loc){
d['E'] = loc.x
d['F'] = loc.y
println d['C']+" "+loc.x+" "+loc.y
}
}
// if(!d['E']){
// def loc = getAmapLocation("ÁúÑÒÊÐ", d['B'], d['C'])
// if(loc){
// d['E'] = loc.x
// d['F'] = loc.y
// }
// }
// if(!d['E']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def loc = getAmapOldLocation(dist.city, dist.region, d['C'])
// if(loc){
// d['E'] = loc.x
// d['F'] = loc.y
// }
// }
//
// }
});
}
def getLocation(city, region, name){
String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
loc = data.location
}
})
return loc;
}
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
if(name == data.name){
loc = data.location
}
}
})
return loc;
}
def getAmapOldLocation(city, region, name){
String sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
if(name == data.name){
loc = data.location
}
}
})
return loc;
}
}
@@ -1,8 +1,10 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
@@ -43,13 +45,11 @@ class ExportPgsqlEvaluationResult {
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
//
def curDate = curMonth.toString()+"28"
// ÉúłÉMD5ÎÄźţ
exportMd5()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// okÎÄźţ
exportOkFile()
exportInOk()
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
}
def exportAvmEvaluationGJNB(yearMonth){
@@ -363,20 +363,31 @@ class ExportPgsqlEvaluationResult {
}
def exportOkFile(){
def dataCodeFileName = linux_prefix + "${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = win_prefix + "${curDate}.ok"
def exportInOk(){
Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
}
}
createFileToFtp("ok txt", [], dataCodeFileName, curDate, false)
def fileName = linux_prefix + "/${curDate}.ok"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "${curDate}.ok"
}
createFileToFtp("ok txt",fileRows, fileName, curDate, false)
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" Êý¾ÝÎļþÊä³öÍê³É......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
// println tableName+" ÉĎ´ŤFtpÍęłÉ......"
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
println tableName+" ÉĎ´ŤFtpÍęłÉ......"
}
}
@@ -1,14 +1,618 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
class ExportPgsqlRonghe {
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres",
"postgres", "postgres", "org.postgresql.Driver")
def citys = []
def database = "base."
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def linux_prefix = "/ccb_ronghe/${curDate}/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\"
static main(args) {
def task = new ExportPgsqlRonghe()
task.run(args)
}
def run(args) {
def new_dists = []
citys = CcbCommonUtils.getRunningCitys(args)
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date() + 1, -1), "yyyy-MM-dd"), new_dists)
if (curDate.toString().endsWith("10")) {
exportEvaluateRate()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportBigdataCommunityGJRH()//
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
exportBigdataCommhouseuseGJ()//
exportBigdataPositionGJ()//
exportBigdataDistrictFJW()//
exportBigdataAliasFJW()//
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
// exportBigdataCityPrice(queryDate)
// exportBigdataDistPrice(queryDate)
// exportBigdataCommPrice(queryDate)
exportInOk()
// }
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
}
}
def exportBigdataDealFJW2(curDate, dealdate, new_dists){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("10")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')";
allRows.addAll(dealData(sql))
}
}
}
createFileToFtp("BIGDATA_DEAL_FJW", allRows, fileName, curDate, false)
}
def dealData(def sql){
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.urbn_id
data << r.mdl_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_prd_val
data << r.gujia_wbt_floor
data << r.gujia_lyr_tp_val
data << r.gujia_tot_floor_num
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.gujiaconstructionarea
data << r.gujia_set_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_room
data << r.gujia_dept
data << r.gujia_ktch_num
data << r.gujia_toliet
data << r.gujia_hs_use
data << r.gujia_indr_stc_val
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujia_ases_bt_ln_num
data << r.gujia_trdtckt_prc
data << r.gujia_mdl_tot_prc
data << r.gujia_dcl_tot_prc
data << r.gujia_ases_tot_prc
data << r.gujiasigntonlinectrid
data << r.txn_dt
data << r.gujia_txn_tp_val
data << r.gujia_sell_pymtmod
data << r.gujia_mdl_cyc
data << r.gujiawthrext_lift_ind
data << r.gujiagvlndfargrdnarea
data << r.gujia_gv_lu_tbl_area
data << r.gujia_gv_sun_tbl_area
data << r.gujiagvbaywindow_area
data << r.gujiagvarea_tower_flr
data << r.gujia_gv_grg_area
data << r.gujiagvundgdroom_area
data << r.gujia_gv_othr_area
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_mkdam
data << r.gujia_data_tm
allRows << data
});
return allRows
}
def exportEvaluateRate(){
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from t0831_evaluate_rate_fjw_w";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_ceft_id
data << r.gujia_ceft_tp
data << r.gujia_ceft
data << r.gujia_seq
data << r.gujia_urbn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
def fileName = linux_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("EVALUATE_RATE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataBuildprojFjw(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_BUILDPROJ_FJW", [], fileName, curDate, false)
}
def exportBigdataCommlayoutpicFjw(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMLAYOUTPIC_FJW", [], fileName, curDate, false)
}
def exportBigdataCommunityGJRH(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,comm_nm,gujia_comm_fmnm,gujia_admndiv_id,gujia_str,gujia_hsno,gujia_dtl_adr,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_comm_tp_val,gujiawthr_new_pan_ind,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_comm_intd,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujiaprhsld_hspty_prc,gujiacomm_tot_rsd_num,gujiacommrntfee_avprc,gujia_comm_avprc from "+database+"t0831_bigdata_community_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_comm_fmnm
data << r.gujia_admndiv_id
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_dtl_adr
data << r.gujiaocpoflandtotarea
data << r.gujia_bld_tot_area
data << r.gujia_comm_tp_val
data << r.gujiawthr_new_pan_ind
data << r.gujia_grnrate
data << r.gujia_vol_rate
data << r.gujia_park_num
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
createFileToFtp("BIGDATA_COMMUNITY_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommbuildstruGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id, gujia_comm_id, gujia_bld_stc, rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+"t0831_bigdata_commbuildstru_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_bld_stc
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMBUILDSTRU_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommfeeGj(){
def allRows = []
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,co_id,gujia_eps_val,gujia_hs_use,gujia_bld_tp_val,gujia_comm_eps_tp,gujia_eps_cmnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commfee_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.co_id
data << r.gujia_eps_val
data << r.gujia_hs_use
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_comm_eps_tp
data << r.gujia_eps_cmnt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMFEE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommhousetypeGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_comm_hs_tp,rmrk_dsc,to_char(data_crt_tm'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhousetype_fjw_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_comm_hs_tp
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMHOUSETYPE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommbuildyearGj(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_bld_yr_rc,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commbuildyear_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_bld_yr_rc
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMBUILDYEAR_FJW", allRows, fileName, curDate, false)
}
def exportBigdataCommhouseuseGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_hs_use,gujia_ladder_accnum,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhouseuse_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_hs_use
data << r.gujia_ladder_accnum
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMHOUSEUSE_FJW", allRows, fileName, curDate, false)
}
def exportBigdataPositionGJ(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_obj_tp,gujia_obj_id,gujia_lo_tp,gujia_sn,lgt,ltt,gujia_hignas_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_position_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_obj_tp
data << r.gujia_obj_id
data << r.gujia_lo_tp
data << r.gujia_sn
data << r.lgt
data << r.ltt
data << r.gujia_hignas_val
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_POSITION_FJW", allRows, fileName, curDate, false)
}
def exportBigdataDistrictFJW(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_district_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_admndiv_nm_cpa
data << r.urbn_id
data << r.gujia_natstd_cd
data << r.gujia_dspl_seq
data << r.gujia_admndiv_cgy
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_DISTRICT_FJW", allRows, fileName, curDate, false)
}
def exportBigdataAliasFJW(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,rel_id,gujia_comm_id,comm_nm,gujia_pcsst,gujia_vld_dt,gujia_vrsn_dsc,gujia_wthr_prim_nm,gujia_dtsrc,gujia_dsc,gujia_als_lvl,gujia_adr_tp_ecd,gujia_wthr_adt_ind,gujia_wthr_dspl_ind from "+database+"t0831_address_alias_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.rel_id
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_pcsst
data << r.gujia_vld_dt
data << r.gujia_vrsn_dsc
data << r.gujia_wthr_prim_nm
data << r.gujia_dtsrc
data << r.gujia_dsc
data << r.gujia_als_lvl
data << r.gujia_adr_tp_ecd
data << r.gujia_wthr_adt_ind
data << r.gujia_wthr_dspl_ind
allRows << data
});
}
def fileName = linux_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
createFileToFtp("ADDRESS_ALIAS_FJW", allRows, fileName, curDate, false)
}
def exportBigdataBuildingFJW(){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
def index = 0
for(def city : citys) {
def allRows = []
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_bld_id,bld_id,gujia_bld_nm,gujia_bld_als_dsc,gujia_admndiv_id,gujia_comm_id,gujia_bld_yr_rc,gujia_bld_tp_val,gujia_bld_stc,floor_num,gujia_unit_num,gujia_accnum_val,gujiawthrext_lift_ind,gujia_lift_num,gujiajiladderjiactval,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_vld_ind from "+database+"t0831_bigdata_building_fjw_w where urbn_id='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_bld_als_dsc
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_bld_yr_rc
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.floor_num
data << r.gujia_unit_num
data << r.gujia_accnum_val
data << r.gujiawthrext_lift_ind
data << r.gujia_lift_num
data << r.gujiajiladderjiactval
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_vld_ind
allRows << data
});
if(allRows && allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
def exportBigdataCityPrice(queryDate){
def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from t0831_city_price_gj_h where CITYID='${cityCode}'";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.SALEPRICE?r.SALEPRICE:''
data << r.RENTPRICE?r.RENTPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.HOUSEUSE
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.DATATYPE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.SOURCECODE
allRows << data
});
}
createFileToFtp("BIGDATA_CITYPRICE", allRows, fileName, curDate, false)
}
def exportInOk(){
Vector files = CcbFtpTestUtils.listFileNames("rh", curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate)
}
}
def fileName = linux_prefix+"${curDate}.ok"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "${curDate}.ok"
}
createFileToFtp("融合统计文件长度ok文件", fileRows, fileName, curDate, false)
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println tableName+" 上传Ftp完成......"
}
}
@@ -27,7 +27,7 @@ class BigdataCitypriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def curRunDate = 202212//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
static main(args) {
runBatch(new BigdataCitypriceBatch(), args)
@@ -36,8 +36,8 @@ class BigdataCitypriceBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def startDate = 20221201//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def endDate = 20221231//DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
cityRun(citys, startDate, endDate, cityCodeDataMap)
@@ -29,6 +29,10 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
citys = CcbCommonUtils.getRunningCitys(args)
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
@@ -53,7 +57,11 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataCommlayoutpicFjw()
exportEvaluateRate()
// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
exportThirdPartyPrice2(thirdPartyMonth, new_dists)
@@ -65,7 +73,11 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataSupplierStatistics(lastMonth)
exportBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出
}
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
@@ -1141,6 +1153,147 @@ class ExportDistrictRongHe extends BaseBatch{
println "bigdata_modeling_base ÉÏ´«FtpÍê³É......"
}
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
// def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";//
def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.SALEPRICE?r.SALEPRICE:''
data << r.RENTPRICE?r.RENTPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.HOUSEUSE
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.DATATYPE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.SOURCECODE
allRows << data
});
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_CITYPRICE 上传Ftp完成......"
}
def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate}
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.DISTID
data << r.SALEPRICE?r.SALEPRICE:''
data << r.RENTPRICE?r.RENTPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.HOUSEUSE
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.DATATYPE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.SOURCECODE
allRows << data
});
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DISTPRICE 上传Ftp完成......"
}
def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";//
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.YMID
data << r.COMMID
data << r.SALEPRICE?r.SALEPRICE:''
if(r.SALECOUNT>0) {
data << r.SALECOUNT
}else{
data << ''
}
data << r.RENTPRICE?r.RENTPRICE:''
if(r.RENTCOUNT>0){
data << r.RENTCOUNT
}else{
data << ''
}
data << r.UPDATETIME
data << r.SALEBLDGAREA?r.SALEBLDGAREA:''
data << r.RENTBLDGAREA?r.RENTBLDGAREA:''
data << r.SALETOTALPRICE?r.SALETOTALPRICE:''
data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:''
data << r.HOUSEUSE
data << r.SDATERANGE
data << r.RDATERANGE
data << r.DATATYPE
data << r.SOURCECODE
allRows << data
});
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMPRICE 上传Ftp完成......"
}
def exportBigdataCommlayoutpicFjw(){
@@ -1154,7 +1307,132 @@ class ExportDistrictRongHe extends BaseBatch{
println "BIGDATA_COMMLAYOUTPIC_FJW ÉÏ´«FtpÍê³É......"
}
def exportOk(curDate){
def exportBigdataForsale(publishDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
def index = 0
for(def city : citys){
def cityCode = cityCodeMap.get(city)
if(city=="苏州"){
continue
}
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " +
"where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')";
allRows = getHouseDatas(sql)
}
if(allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " +
"from bigdata_forsale_gj where distid in ${newstr}"
def rets2 = getHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true);
}
}
println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def getHouseDatas(sql){
def allRows = []
def index = 0
sqlInstance.eachRow(sql,{r->
index = index+1
println r.CITYNAME +" "+index
def data = []
data << r.CITYID
data << r.FORSALEID
data << r.PROPCODE
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.UNITPRICE
data << r.TOTALPRICE
data << r.PAYTYPE
data << r.FLOORNUM
data << r.FLOORCNT
data << r.BUILDTYPE
data << r.HOUSESTRU
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEASPECT
data << r.FITMENT
data << r.IFELEVATOR
data << r.ELEVATORHOUSE
data << r.PUBLISHTIME
data << r.EXPIRETIME
data << r.IFFIRST
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.PUBLISHER
data << r.PHONE
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
// data << r.ATTACH //融合房源没有这个字段
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
// allRows.clear()
// }
});
return allRows
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
@@ -0,0 +1,82 @@
package com.fangjia.script.chongqing
import com.sun.org.apache.xml.internal.security.utils.Base64;
import org.apache.commons.lang3.StringUtils;
import javax.crypto.Cipher;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.DESedeKeySpec;
import javax.crypto.spec.IvParameterSpec;
import java.security.Key;
class Des3 {
private final static String encoding = "UTF-8";
static main(args) {
def task = new Des3()
task.run()
}
def run(){
try {
String key = "111036369260679051122113";
String iv = "23235645";//IV length must be 8 bytes long
//加密
String encryptStr = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"//encode("test", key, iv);
System.out.println("密文: " + encryptStr);
//解密
String decryptStr = decode(encryptStr, key, iv);
System.out.println("明文: " + decryptStr);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 方法描述:3DES加密
* @author hyang
*
* @param plainText 明文
* @param secretKey 密钥
* @param iv 加密向量
* @return String 密文
* @throws Exception
*/
public static String encode(String plainText, String secretKey, String iv)
throws Exception {
Key deskey = null;
DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes());
SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede");
deskey = keyfactory.generateSecret(spec);
Cipher cipher = Cipher.getInstance("desede/CBC/PKCS5Padding");
IvParameterSpec ips = new IvParameterSpec(iv.getBytes());
cipher.init(Cipher.ENCRYPT_MODE, deskey, ips);
byte[] encryptData = cipher.doFinal(plainText.getBytes(encoding));
return Base64.encode(encryptData);
}
/**
* 方法描述: 3DES解密
* @author hyang
*
* @param encryptText 密文
* @param secretKey 密钥
* @param iv 加密向量
* @return String 明文
* @throws Exception
*/
public static String decode(String encryptText, String secretKey, String iv)
throws Exception {
if (StringUtils.isBlank(encryptText) || StringUtils.isBlank(secretKey)) {
return "";
}
Key deskey = null;
DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes());
SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede");
deskey = keyfactory.generateSecret(spec);
Cipher cipher = Cipher.getInstance("desede/CBC/NoPadding"); //desede/CBC/NoPadding 无填充 desede/CBC/PKCS5Padding 加密需要填充,解密一般加不加都结果一致
IvParameterSpec ips = new IvParameterSpec(iv.getBytes());
cipher.init(Cipher.DECRYPT_MODE, deskey, ips);
byte[] decryptData = cipher.doFinal(Base64.decode(encryptText));
return new String(decryptData, encoding).trim();
}
}
@@ -0,0 +1,100 @@
package com.fangjia.script.chongqing
import com.fangjia.SecretUtils
import javax.crypto.Cipher
import javax.crypto.SecretKey
import javax.crypto.spec.SecretKeySpec
class Des3Test {
//定义加密算法,有DES、DESede(即3DES)、Blowfish
private static final String Algorithm = "DESede";
private static final String PASSWORD_CRYPT_KEY = "111036369260679051122113";
/**
* 加密方法
* @param src 源数据的字节数组
* @return
*/
public static byte[] encryptMode(byte[] src) {
try {
SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); //生成密钥
Cipher c1 = Cipher.getInstance(Algorithm); //实例化负责加密/解密的Cipher工具类
c1.init(Cipher.ENCRYPT_MODE, deskey); //初始化为加密模式
return c1.doFinal(src);
} catch (java.security.NoSuchAlgorithmException e1) {
e1.printStackTrace();
} catch (javax.crypto.NoSuchPaddingException e2) {
e2.printStackTrace();
} catch (java.lang.Exception e3) {
e3.printStackTrace();
}
return null;
}
/**
* 解密函数
* @param src 密文的字节数组
* @return
*/
public static byte[] decryptMode(byte[] src) {
try {
SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm);
Cipher c1 = Cipher.getInstance(Algorithm);
c1.init(Cipher.DECRYPT_MODE, deskey); //初始化为解密模式
return c1.doFinal(src);
} catch (java.security.NoSuchAlgorithmException e1) {
e1.printStackTrace();
} catch (javax.crypto.NoSuchPaddingException e2) {
e2.printStackTrace();
} catch (java.lang.Exception e3) {
e3.printStackTrace();
}
return null;
}
/*
* 根据字符串生成密钥字节数组
* @param keyStr 密钥字符串
* @return
* @throws UnsupportedEncodingException
*/
public static byte[] build3DesKey(String keyStr) throws UnsupportedEncodingException{
byte[] key = new byte[24]; //声明一个24位的字节数组,默认里面都是0
byte[] temp = keyStr.getBytes("UTF-8"); //将字符串转成字节数组
/*
* 执行数组拷贝
* System.arraycopy(源数组,从源数组哪里开始拷贝,目标数组,拷贝多少位)
*/
if(key.length > temp.length){
//如果temp不够24位,则拷贝temp数组整个长度的内容到key数组中
System.arraycopy(temp, 0, key, 0, temp.length);
}else{
//如果temp大于24位,则拷贝temp数组24个长度的内容到key数组中
System.arraycopy(temp, 0, key, 0, key.length);
}
return key;
}
/**
* @param args
*/
public static void main(String[] args) {
String msg = "3DES加密解密案例";
System.out.println("【加密前】:" + msg);
//加密
// byte[] secretArr = Des3Test.encryptMode(msg.getBytes());
// System.out.println("【加密后】:" + new String(secretArr));
String s = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"
//解密
byte[] myMsgArr = Des3Test.decryptMode(s.getBytes("UTF-8"));
System.out.println("【解密后】:" + new String(myMsgArr));
}
}
@@ -0,0 +1,703 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
class HailanTest {
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new HailanTest();
task.run();
}
def run(){
def inputfile = "D:\\数据\\海南\\统计.xlsx"
// match(inputfile)
// addPrice(inputfile)
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
// addHouse(inputfile)
// stat(inputfile)
hailanStat(inputfile)
}
def addHouse(inputfile){
def datas = []
datas << ["类型(住宅/别墅)","行政区划","详细地址","房号","小区名称","建筑面积","所在楼层","详细地址","小区名","交易日居住用房价格指数","评估日居住用房价格指数","是否有电梯","所在楼层","总楼层","内部装修","平面布置","朝向",
"景观","建筑结构","建筑面积(平方米)","房屋可使用年限","房屋已使用年限","评估价(单价)","挂牌价(单价)","成交价(单价)","小区/楼盘均价(单价)"]
def index = 0
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['L']){
index ++
def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district")
def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k'])
println index +" "+ row['C']+" "+ rets.size()
if(rets){
def lastRets = []
for(def r : rets){
def fjwPrice = row['M']
if(fjwPrice){
def precent = java.lang.Math.abs((fjwPrice-r.avg_price)*100/r.avg_price)
if(precent>=10){
continue;
}
}
if(row['A'] && row['A']=="别墅"){
def inputAare = row['F'].toString().toDouble()
def minArea = inputAare + 50
def maxArea = inputAare + 100
if(r.area<minArea && r.area>maxArea){
continue;
}
}
lastRets << r
if(lastRets.size()>=3){
break;
}
}
if(lastRets){
for(def lr : lastRets){
def data = []
data << row['A']
data << row['B']
data << row['C']
data << row['D']
data << row['E']
data << row['F']
data << row['G']
if(dist){
data << dist.address
}else{
data << ""
}
if(dist){
data << dist.name
}else{
data << ""
}
data << ""
data << ""
if(lr.height){
if(lr.height>6){
data << "是"
}else{
data << "否"
}
}else{
data << ""
}
data << lr.floor
data << lr.height
data << lr.fitment
data << ""
data << lr.direction
data << ""
data << lr.build_type
data << lr.area
data << dist.property_life
// def year = dist.create_year
// if(year){
// data << getHouseYear(year.toInteger())//已使用年限
// }else{
// data << ""//已使用年限
// }
data << getHouseYear(dist.create_year , lr.build_date, dist.build_date)
data << getEvalutaionPrice(lr, dist)//评估价(单价)
data << lr.avg_price
data << getDealPrice(dist)//成交单价
data << row['M']
datas << data
}
}
}
}
})
ExcelUtils.write("D:\\数据\\海南\\海南样本数据.xlsx", datas, "样本数据")
}
def getDealPrice(dist){
def json = OpenApiClient.searchDealPrice(dist._id.toString(),"20220101", 100)
def prices = []
if(json && json.code==200){
def result = json.result
if(result){
for(def r : result){
prices << r.avgPrice
}
}
}
def dealAvgPrice = null
if(prices){
dealAvgPrice = (prices.sum()/prices.size()).toInteger()
}
return dealAvgPrice;
}
def getHouseYear(create_year , house_build_date, dist_build_date){
def year = null
if(house_build_date){
year = getYear(house_build_date)
}else if(dist_build_date){
year = getYear(dist_build_date)
}else if(create_year){
year = create_year
}
if(year){
year = year.toInteger()
def curYear = new Date().getYear()+1900
def modYear = curYear - year
return modYear.toInteger().toString()
}
return null
}
def getYear(finishBuildingDate){
if(finishBuildingDate){
Pattern pat3 = Pattern.compile("\\d{4}");
def matcher3 = pat3.matcher(finishBuildingDate);
if(matcher3.find()){
return matcher3.group()
}
}
return null
}
def getEvalutaionPrice(house, dist){
def params = [:]
params.put("city", dist.city);
params.put("district", dist.region);
params.put("name", dist.name);
params.put("size", String.valueOf(house.area));
if(house.height) {
params.put("maxFloor", dist.house_height.toString());
params.put("direction", getDirectionCode(house.direction).toString());//朝向
if (house.house_floor) {
params.put("floor", house.house_floor.toString());
}
}
def fitmentCode = null
if(house.fitment){
fitmentCode = getFitmentCode(house.fitment)
params.put("decoration", fitmentCode.toString());//装修
}
params.put("scope", "1")
def result = OpenApiClient.evaluateDistrictPrice(params);
if(result){
return result.avgPrice
}
return null
}
def getDirectionCode(def s){
def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10]
return directMap.get(s);
}
def getFitmentCode(def s){
if(s){
if(["毛坯"].contains(s)){
return 1;
}else if(["简单装修","简装"].contains(s)){
return 2;
}else if(["精装修","精装"].contains(s)){
return 3;
}else if(["豪华装修","豪装"].contains(s)){
return 4;
}
}
return null
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
// if(vallaFlag){
// continue;
// }
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
// if(!seawater.height){
// seawater.height = null
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
// if(!seawater.room){
// seawater.room = null
// }
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
rowMap.houseId = seawater.id
// rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.address = seawater.address
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.build_type = seawater.build_type
rowMap.build_date = seawater.build_date
rowMap.district_name = seawater.district_name
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rets << rowMap
}
}
return rets;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
def addCommId(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['H']){
def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district")
if(dist){
row['L'] = dist._id.toString()
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['L']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['M'] = priceObj.avg_price
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C']){
String city = row['H']//RegionData.formatCity(row['B'])
if(city){
def address = row['C']
// def region = row['C']
println "[match] city:"+city+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E'])
if(jsonObject != null && jsonObject.precise!=-1) {
row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("name")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("id")
}
}else{
println row['C']
}
}
})
}
def stat(path){
ExcelUtils.modify(path, { row ->
if(row['L'] && !row['H']){
def city = row['L']
def region = row['M']
def dists = null
if(city && region){
dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias","estate_type2"])
}else{
dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias","estate_type2"])
}
def distLists = []
def villaLists = []
for(def d : dists){
if(d.estate_type2=="别墅"){
villaLists << d
}else{
distLists << d
}
}
row['J'] = dists.size()
def haveHouseList = findHaveHouseDist(distLists)
def haveVillaList = findHaveHouseDist(villaLists)
row['H'] = haveHouseList
row['I'] = haveVillaList
// row['N'] = distLists.size()
// row['O'] = villaLists.size()
}
})
}
def hailanStat(path){
def total = 0
def distTotal = 0
def villaTotal = 0
def haveHouseDistTotal = 0
def haveHouseVillaTotal = 0
def buildStructureTotal = 0
// ExcelUtils.modify(path, { row ->
// if (row['L'] && !row['K']) {
//"海口","三亚","三沙","儋州",
def citys = ["五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
for(def city : citys) {
def dists = null
// if (city && region) {
// dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city", "region", "name", "alias", "estate_type2", "build_structure"])
// } else {
dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias", "estate_type2", "build_structure"])
// }
def distLists = []
def villaLists = []
for (def d : dists) {
if (d.estate_type2 == "别墅") {
villaLists << d
} else {
distLists << d
}
}
total = total + dists.size()
// def structureCount = 0
// for(def d : dists){
// if(d.build_structure){
// structureCount ++
// }
// }
// println structureCount
// row['K'] = structureCount
// buildStructureTotal = buildStructureTotal + structureCount
//
def haveHouseList = findHaveHouseDist(distLists)
def haveVillaList = findHaveHouseDist(villaLists)
haveHouseDistTotal = haveHouseDistTotal + haveHouseList
haveHouseVillaTotal = haveHouseVillaTotal + haveVillaList
distTotal = distTotal + distLists.size()
villaTotal = villaTotal + villaLists.size()
// println c +" "+total+" "+buildStructureTotal
// }
}
// })
println total +" "+haveHouseDistTotal+" "+haveHouseVillaTotal+" "+distTotal+" "+villaTotal+" "+buildStructureTotal
}
def findHaveHouseDist(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println "小区量:"+districts.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(dist.city, dist.region, "sell", 20210101, null, dist.name, dist.alias, dist.estate_type2)
if(!ret){
ret = findThirdPrice(dist)
}
if(ret && ret.size()>0){
rets.add(dist)
}
println totalCount.get() +" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets.size()
}
def findThirdPrice(dist){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("name", dist.name)
q.put("s_date", new BasicDBObject('$gte', 202201))
def dbcursor = AVERAGE_STATISTICS.find(q)
def prices = []
if(dbcursor){
for(def d : dbcursor){
def set = d.keySet();
Iterator it = set.iterator()
while(it.hasNext()){
def key = it.next()
if(key && key.contains("_price") && key!="fangjia_price" && key!="last_price"){
def price = d[key]
if(price){
prices << price
}
}
}
}
}
return prices;
}
}
@@ -0,0 +1,328 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.apache.lucene.queryparser.classic.QueryParser
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class MingShenPocStat {
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
private static final DBCollection district_sell_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
static main(args) {
def task = new MingShenPocStat()
// task.run()
task.officeRun()
}
def officeRun(){
def path = "D:\\数据\\民生\\抽样-写字楼类.xlsx"
def datas = []
ExcelUtils.modify(path, { d ->
if(d['B']){
def office = CommonDatautils.findDistrictById(d['C'])
if(office){
def alias = []
alias << office.name
if(office.alias){
def nameAlias = office.alias
for(def o : nameAlias){
if(o){
alias << o
}
}
}
def aliasStr = escapeExprSpecialWord(alias.join("','"))
def houses = findOfficeHouseCount(office.city, office.region, aliasStr)
if(houses){
if(houses.size()>=2){
houses = houses[0..1]
}
for(def h : houses){
def obj = findOfficeAvgPrice(office)
def rets = []
rets << d['A']
rets << d['B']
rets << ''
rets << h.region
rets << office.address
rets << h.area
rets << h.floor
rets << h.height
def level = office.bed_kind
rets << level
rets << h.fitment
def date = office.finish_building_date
if(date){
rets << date.substring(0, 4)
}else{
rets << ""
}
rets << ""
rets << ""
def price = ""
if(obj && obj.avg_price){
price = obj.avg_price
rets << obj.avg_price
}else{
rets << ""
}
if(price){
rets << (price*h.area).toInteger()
}
datas << rets
}
}
}else{
def rets = []
rets << d['A']
rets << d['B']
datas << rets
}
// def districts = CommonDatautils.findAllDistrictByCity(d['C'],"office",["city", "region", "name", "alias","address","bed_kind","finish_building_date"])
}
})
ExcelUtils.write("D:\\数据\\民生\\抽样-写字楼类-样本.xlsx", datas, "样本")
}
def run(){
def path = "D:\\数据\\民生\\样本字段覆盖率.xlsx"
// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\二期估值数据每周统计\\二期估值数据每周统计\\估值数据统计结果20230112.xlsx", 0 , 1)
//
// ExcelUtils.modify(path, { row ->
// if(row['C']) {
// def total = 0
// def pre = 0
// for(def d : datas){
// if(row['C']==d['A']){
// total = d['B']
// pre = d['H']
// break
// }
// }
// if(total){
// def c = (total*pre/100).toInteger()
// row['E'] = c
// row['F'] = pre+"%"
// }
// }
// })
def totalcount = findHouseTotal("大连")
def count = findHouseCount("大连", "height")
println DataHandleUtils.getDoubleRound(2, count*100/totalcount)
// ExcelUtils.multiThreadingModify(path, { row ->
// if(row['C']) {
//def districts = CommonDatautils.findAllDistrictByCity(row['C'],"district",["city", "region", "name", "alias"])
// def districts = CommonDatautils.findDistrictByCityAndRegions("大连","district",["旅顺口"], ["city", "region", "name", "alias"])
// def total = districts.size()
// if(total){
// def house_dist_count = findHouseStat(districts)
// def pre = house_dist_count*100/total
// pre = DataHandleUtils.getDoubleRound(2, pre)
// println total+" "+house_dist_count+" "+pre
// }
// def count = findHouseCount(row['C'])
// row['D'] = count
// println row['C']+" "+count
// row['H'] = findDistrictSellStat(row['C'])
// row['G'] = getNewDist(row['C'])
// def totalcount = findHouseTotal(row['C'])
// def total = row['Z']
// def count = findHouseCount(row['C'], "build_date")
// row['Q'] = DataHandleUtils.getDoubleRound(2, count*100/total)
// println count +" "+row['Q']
// def obj = getFieldPrecent(row['C'])
// if(obj){
// def detail = obj.district_detail
// row['E'] = detail.alias.toDouble()*100
// row['F'] = detail.address.toDouble()*100
// row['G'] = detail.lng.toDouble()*100
// row['H'] = detail.buildType.toDouble()*100
// row['I'] = detail.outPics.toDouble()*100
// row['J'] = detail.developer.toDouble()*100
// row['Q'] = detail.buildDate.toDouble()*100
// row['R'] = detail.buildDate.toDouble()*100
// }
// }
// });
}
def getFieldPrecent(city){
def q = new BasicDBObject()
q.put("city", city)
def rets = CITY_INFO_CONN.find(q).sort(new BasicDBObject('weekend', -1))
if(rets){
return rets[0]
}
return null
}
def getNewDist(city){
String sql = "select count(*) as count from community where city='${city}' and source='贝壳找房-新房' "
def count = null
ElasticUtils.executeSqlQuery(sql,{ data->
count = data.count
})
return count;
}
def findDistrictSellStat(def city){
def q = new BasicDBObject()
q.put("city", city)
def count = district_sell_stat_coll.count(q)
return count
}
def findHouseStat(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println "小区量:"+districts.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(rMap.city, rMap.region, "sell", 20210101,rMap.name, rMap.alias)
if(ret && ret.size()>0){
rets.add(rMap)
}
println totalCount.get() +" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets.size()
}
def findHouseTotal(city){
def count = 0
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20220101 and city='${city}'", {
count = it.count
println("$city\t${it.count}")
})
return count
}
def findHouseCount(city, field){
def count = 0
def sql = "select count(*) as count from seawater where city='${city}' and cat='sell' and "+field+" is not null"
ElasticUtils.executeSqlQuery(sql, {
count = it.count
println("$city\t${it.count}")
})
return count
}
def findHouses(def city, def region, def cat, def startDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses;
}
def findOfficeAvgPrice(office){
def sql = "select * from office where city='${office.city}' and region='${office.region}' and name='${office.name}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, {d->
rets << d
})
if(rets){
def obj = rets[0]
return obj
}
return null
}
def findOfficeHouseCount(city, region, names){
def sql = "select * from seawater where city='${city}' and region='${region}' and cat='office_sell' and district_name in ('${names}') and root='58.com'"
def rets = []
ElasticUtils.executeSqlQuery(sql, {d->
rets << d
})
return rets
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
}
@@ -0,0 +1,459 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class MingShenPocTest {
def total = 5000
static main(args) {
def task = new MingShenPocTest()
task.run()
}
def run(){
def path = "D:\\数据\\民生\\poc测试.xlsx"
//9级标准化
//structureV2(path)//针对海南poc地址数据此接口不合适使用
//省市区拆分
// ner(path)//城市,区域带市带区
// match(path)
// addPrice(path)
// addLatLng(path)
// addAroundPrice(path)
// addAuthor(path)
addHouse(path)
}
def structureV2(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A']){
def jsonObj = FangjiaClientUtils.addressStructure(null, row['A'])
if(jsonObj && jsonObj.result){
if(jsonObj.result.province){
row['B'] = jsonObj.result.province
}
if(jsonObj.result.city){
def cityStr = null
if(jsonObj.result.city=="市辖区"){
cityStr = jsonObj.result.province
}else{
cityStr = jsonObj.result.city
}
row['C'] = RegionData.formatCity(cityStr)
}
if(jsonObj.result.district){
row['D'] = jsonObj.result.district
}
}
}
})
}
def ner(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['A'] && !d['B']) {
def json = FangjiaClientUtils.addressStand(null, d['A'])
if(json && json.entities){
def entities = json.entities
for(def m : entities){
if(m.type=="prov"){
d['B'] = m.word
}
if(m.type=="city"){
def cityStr = m.word
if(cityStr){
d['C'] = cityStr//RegionData.formatCity(cityStr)
}
}
if(m.type=="dist") {
if(d['C']){
d['D'] = m.word //RegionData.formatRegion(d['C'], m.word)
}
}
}
}
}
});
}
//可以根据测试数据添加 推荐楼盘名
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C'] && !row['E']){
def city = RegionData.formatCity(row['C'])
def region = null
if(city){
region = RegionData.formatRegion(city, row['D'])
}
def address = row['A']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, null)//"property"
if(jsonObject != null && jsonObject.precise!=-1) {
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("propertyType")
row['I'] = jsonObject.get("id")
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['I']
if(commid && !row['J']){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def house = [:]
house.area = 100
def price = getEvalutaionPrice(house, dist)
if(price){
row['J'] = price
}
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
// if(priceObj){
// row['J'] = priceObj.avg_price
// }
}
}
})
}
def getEvalutaionPrice(house, dist){
def params = [:]
params.put("city", dist.city);
params.put("district", dist.region);
params.put("name", dist.name);
params.put("size", String.valueOf(house.area));
if(house.height) {
params.put("maxFloor", dist.house_height.toString());
params.put("direction", getDirectionCode(house.direction).toString());//朝向
if (house.house_floor) {
params.put("floor", house.house_floor.toString());
}
}
def fitmentCode = null
if(house.fitment){
fitmentCode = getFitmentCode(house.fitment)
params.put("decoration", fitmentCode.toString());//装修
}
params.put("scope", "1")
def result = OpenApiClient.evaluateDistrictPrice(params);
if(result){
return result.avgPrice
}
return null
}
def getDirectionCode(def s){
def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10]
return directMap.get(s);
}
def getFitmentCode(def s){
if(s){
if(["毛坯"].contains(s)){
return 1;
}else if(["简单装修","简装"].contains(s)){
return 2;
}else if(["精装修","精装"].contains(s)){
return 3;
}else if(["豪华装修","豪装"].contains(s)){
return 4;
}
}
return null
}
def addLatLng(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['J'] && d['H'] && (d['H']=="普通住宅" || d['H']=="商住" || d['H']=="别墅")) {
def json = BaiduMapUtils.searchByAddress(d['A'])
if(json && json.location){
d['K'] = json.location.lng
d['L'] = json.location.lat
}
}
})
}
def addAroundPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['J'] && d['C'] && d['K']){
def city = RegionData.formatCity(d['C'])
def dist = new BasicDBObject()
dist.city = city
dist.lng = d['K'].toString().toDouble()*10000000000L
dist.lat = d['L'].toString().toDouble()*10000000000L
dist.estateType = "普通住宅"//d['H']
if(dist.lat) {
List<DBObject> list = DbUtils.findAroundDistrictByRadius(dist, 1000)
if (list) {
list = sortByDistance(dist, list);
if (list && list.size() > 10) {
list = list[0..9]
}
if (list) {
def prices = []
for (def s : list) {
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if (priceObj) {
prices << priceObj.avg_price
}
}
if (prices) {
def arround_price = (int) (prices.sum() / prices.size())
d['J'] = arround_price
}
}
}
}
}
});
}
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
}
//排序
Collections.sort(list, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return list;
}
def addAuthor(def path){
def authors = ["董彦均","曹玉彦","付贵生","沈燕芳","Catherine", "王蕾","吉珉", "张宝剑"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['A']){
if(index < avg){
row['M'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def addHouse(path){
def datas = []
ExcelUtils.multiThreadingModify(path, { row ->
def commId= row['E']
if(commId){
def dist = CommonDatautils.findDistrictById(commId)
if(dist){
def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k'])
if(rets){
def data = []
for(def r : rets){
data << r.city
data << r.region
data << r.district_name
data << r.floor
data << r.height
data << r.fitment
data << r.direction
data << r.build_type
data << r.area
data << r.avg_price
datas << data
}
}
}
}
});
ExcelUtils.write("D:\\数据\\民生\\民生样本数据.xlsx", datas, "样本数据")
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
// def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
// if(vallaFlag){
// continue;
// }
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
// if(!seawater.height){
// seawater.height = null
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
// if(!seawater.room){
// seawater.room = null
// }
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
rowMap.houseId = seawater.id
// rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.address = seawater.address
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.build_type = seawater.build_type
rowMap.build_date = seawater.build_date
rowMap.district_name = seawater.district_name
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rets << rowMap
}
}
return rets;
}
}
@@ -0,0 +1,105 @@
package com.fangjia.script.chongqing
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.io.FileUtils
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.util.Base64;
/**
* 3DES加密工具类
* @author qzz
*/
public class ThreeDESUtils {
/**
* 加解密统一编码方式
*/
private final static String ENCODING = "utf-8";
/**
* 加解密方式
*/
private final static String ALGORITHM = "DESede";
/**
*加密模式及填充方式
*/
private final static String PATTERN = "DESede/ECB/pkcs5padding";
/**
* 3DES加密
*
* @param plainText 普通文本
* @param sK 秘钥(24位密码)
* @return
* @throws Exception
*/
public static String encode(String plainText,String sK) throws Exception {
SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM);
// 3DES加密采用pkcs5padding填充
Cipher cipher = Cipher.getInstance(PATTERN);
// 用密匙初始化Cipher对象
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
// 执行加密操作
byte[] encryptData = cipher.doFinal(plainText.getBytes(ENCODING));
return Base64.getEncoder().encodeToString(encryptData);
}
/**
* 3DES解密
*
* @param encryptText 加密文本
* @return
* @throws Exception
*/
public static String decode(String encryptText, String sK) throws Exception {
SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM);
// 3DES加密采用pkcs5padding填充
Cipher cipher = Cipher.getInstance(PATTERN);
// 用密匙初始化Cipher对象
cipher.init(Cipher.DECRYPT_MODE, secretKey);
// 正式执行解密操作
byte[] decryptData = cipher.doFinal(Base64.getDecoder().decode(encryptText));
return new String(decryptData, ENCODING);
}
public static void main(String[] args) throws Exception {
def rets = []
def datas = readTxt("D:\\数据\\民生\\抽样-住宅类(普通住宅、公寓).txt")
def index = 0
for(def d : datas){
index++
def row = []
def str = decode(d,"111036369260679051122113")
println str
def id = str.substring(str.length()-4)
row << id
row << str.substring(0, str.indexOf(id))
rets << row
}
ExcelUtils.write("D:\\数据\\民生\\民生银行测试数据.xlsx", rets, "抽样-住宅类(普通住宅、公寓)")
// ExcelUtils.multiThreadingModify("D:\\数据\\民生\\加密数据.xlsx", { row ->
// //加密
// String str = row['A']
// //解密
// row['B'] = (decode(str,"111036369260679051122113"));
// });
}
public static def readTxt(def file){
def lines = FileUtils.readLines(new File(file))
def ret = [];
for(line in lines) {
ret << line;
}
return ret;
}
}
@@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -18,6 +18,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
//小区月度均价表(带涨跌幅控制)
@BatchOwner(Watcher.GaoJuan)
@@ -78,6 +79,7 @@ class DistrictSellStatBatch extends BaseBatch {
}else{
districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000)
}
print city +" "+districts.size()
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
@@ -420,8 +420,8 @@ class HouseExportBatch extends BaseBatch{
row << seawater.name
row << seawater.url
// row << seawater.p_date
row << DateUtils.getRandomDate("2022-06-01", "2022-06-30")
row << seawater.p_date
// row << DateUtils.getRandomDate("2022-06-01", "2022-06-30")
rets << row
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -51,6 +51,7 @@ class AddAvgpriceBatch {
for(def disKey : res2){
def array = disKey.split("##");
def estateType2 = districtKeysMap.get(disKey)
//发重算均价命令
CommonUtils.recalculationAverage(array[0], array[1], array[2], estateType2, 0, new Date())
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.gj
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
class ShLongShangHangLocation {
static main(args) {
def task = new ShLongShangHangLocation()
task.run()
}
def run(){
def rets = ["上海市黄兴路1599号","上海市杨浦区长阳路1687号A栋","上海市杨浦区佳木思路432号","上海市杨浦区控江路1063号","上海市杨浦区鞍山路100号","上海市杨浦区延吉东路287号","上海市杨浦区殷行路408号","上海市杨浦区闸殷路1555号102室","上海市杨浦区平凉路1500号"]
def map = [:]
for(def r : rets){
def location = getLocation(r)
if(location){
map.put(r, location)
}
}
def titleMap = [0:'C',1:'D',2:'E',3:'F',4:'G',5:'H',6:'I',7:'J',8:'K']
ExcelUtils.modify("D:\\数据\\上海农商行\\网点与居委分布20230118.xlsx",{row->
if(row['B'] && !row['B'].contains("地址") && row['B'].contains('齐齐哈尔路')){
for(def i=0;i<rets.size();i++){
def s = rets.get(i)
def mapLoc = map.get(s)
def location = getLocation("杨浦区"+row['B'])
if(location && mapLoc){
def distance = GeoUtils.calculateDistance(mapLoc.lng.toDouble(), mapLoc.lat.toDouble(), location.lng.toDouble(), location.lat.toDouble())
def title = titleMap.get(i)
row[title] = distance.toInteger()
}
}
println row['B']
}
},0)
}
def getLocation(def address){
def json = GaoDeApiUtils.geo("上海", address)
if (json && json.status == "1") {
def ret = json.geocodes
if (ret) {
def str = ret.get(0).location
def arrs = str.split(",")
return ["lng": arrs[0], "lat": arrs[1]]
}
}
return null
}
}
@@ -0,0 +1,42 @@
package com.fangjia.script.gj.es
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
findLogs()
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size()
def set = new HashSet()
def index = 0
if(houses) {
for (def d : houses) {
def request = d.request
if(request && !d.domain.contains("k8s2.fangjia.com")){
index ++
println "[${index}] "+d.domain +" "+ request
}
}
}
println set
}catch(Exception e){
e.printStackTrace()
}
return total
}
@@ -251,26 +251,29 @@ public class BeiKeCrawlByChrome {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "±´¿ÇÕÒ·¿"
def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
if(detail.total_price){
def detailUrl = e.select(".title a").attr("href")
detail.url = detailUrl
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}catch(Exception ex){
@@ -328,25 +328,29 @@ public class LianjiaCrawlByChrome {
}
detail.source = "链家在线"
detail.url = e.select(".title a").attr("href")
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
if(detail.total_price){
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("¹²") && floorStr.contains("²ã)")) {
floorStr = floorStr.substring(floorStr.indexOf("¹²") + 1, floorStr.indexOf("²ã)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}
}
}catch(Exception ex){
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202211);
def count2 = findTotalCount(source, 202212);
def count1 = findTotalCount(source, 202212);
def count2 = findTotalCount(source, 202301);
println source +" "+count1 +" "+count2
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.util
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.RandomUtil
import com.fangjia.script.addrepo.InvokeHttpRequest;
import groovy.json.JsonSlurper
@@ -103,24 +104,20 @@ class BaiduMapUtils {
int recount = 0;
while(!stop && recount<3) {
try{
def ret = ABuYunUtil.doGet(url, null, null, "utf-8");
if (ret) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if(jsonObj.getInt("status") == 121){
println "key over"
changeAK()
recount++;
HttpRequest request = new HttpRequest();
try{
String result = request.get(url,null,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
if(json.getInt("status") >= 300){
changeAK();
json = searchByAddress(address);
}else{
def result = jsonObj.result;
if(result){
stop = true
return result;
}else{
stop = true
}
return json.result
}
}catch (Exception e){
e.printStackTrace();
}
return null;
}catch (Exception e){
recount++;
println e.getMessage()
@@ -0,0 +1,43 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]]
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
OPEN_URI_LOG.find(and(eq("user_id", "1007713"), eq("request_uri", "/property/evaluate"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
// if (!hashSet.contains(result)) {
def item = [doc._id.toString(), "东莞银行总行", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
// hashSet << result
// } else {
// println("已存在")
// }
}
if (count % 7 == 0) {
hashSet.clear()
}
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "90010d53a16ce9df544acab297645106"
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390"
/**
* 冪帠僅蛌華硊<br>