This commit is contained in:
2023-05-05 15:51:14 +08:00
16 changed files with 954 additions and 113 deletions
Vendored
+7 -7
View File
@@ -2,7 +2,7 @@
pipeline {
agent any
parameters {
choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'], description: '请选择发布的项目')
choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke'], description: '请选择发布的项目')
}
tools {
jdk 'JDK8'
@@ -17,12 +17,12 @@ pipeline {
stage('Maven Build') {
steps {
script {
def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic']
def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke']
for (def project : projects) {
if (params.PROJECT_NAME == project) {
dir("deliverables/${project}") {
withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') {
sh 'mvn clean package -Dmaven.test.skip=true'
sh 'mvn clean package -Dmaven.test.skip=true'
}
}
}
@@ -33,8 +33,8 @@ pipeline {
stage('Docker Build') {
steps {
script {
def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic']
def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static']
def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke']
def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static', 'geluke':"geluke"]
for (def project : projects) {
if (params.PROJECT_NAME == project) {
dir("projects/${imagesName[project]}") {
@@ -53,8 +53,8 @@ pipeline {
stage('K8S Deploy') {
steps {
script {
def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic']
def k8s_config_map = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static']
def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic', 'geluke']
def k8s_config_map = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static', 'geluke':"geluke"]
for (def project : projects) {
if (params.PROJECT_NAME == project) {
dir("deliverables/k8s") {
@@ -1861,6 +1861,44 @@ public class AddressController {
return mav;
}
/**
* 招商证券POC测试接口封装
*/
public Object cmbTestAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/cmbTest", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/cmbTest", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 五级地址标准化接口
*
@@ -208,7 +208,7 @@ public class HouseController {
}
@Authority(role = Role.USER)
@Authority(role = Role.NONE)
public Object similarAction(@Valid CollateralSampleQuery collateralSampleQuery){
RestfulModelAndView mav = new RestfulModelAndView();
@@ -253,7 +253,7 @@ public class HouseController {
/**商铺相似房源接口*/
@Authority(role = Role.USER)
@Authority(role = Role.NONE)
public Object shopSimilarAction(@Valid ShopCollateralSampleQuery query){
RestfulModelAndView mav = new RestfulModelAndView();
ShopCollateralSampleQueryBean queryBean = new ShopCollateralSampleQueryBean();
@@ -39,7 +39,7 @@ public class JnBankController {
private static final Pattern STAGE_PATTERN = Pattern.compile(".*[\\w东南北中西甲乙丙丁一二三四五六七八九十百千\\d]+(期|区)$");
@Authority(role = Role.USER)
@Authority(role = Role.NONE)
public Object searchAction(@Valid JNBankSearchModel model){
RestfulModelAndView mav = new RestfulModelAndView();
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
@@ -78,9 +78,13 @@ class BigdataCommunityBatch extends BaseBatch{
println "txt 生成->导Oracle"
// delPgData()
// saveToPgSql(rows)
delPgData()
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
// saveToPgSql2(datas)
}
def addDeleteDist(dists, ccbMaps, rets){
@@ -255,14 +259,74 @@ class BigdataCommunityBatch extends BaseBatch{
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(20000, sql){ ps->
sqlInstance.withBatch(1, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def saveToPgSql2(rows){
def newRows = []
for(def r : rows) {
def newR = []
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
// r[i] = null
newR << null
}else{
newR << r[i]
}
}
newR << null
newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
newR << null
newR << '2999-12-31'
newR << null
newR << null
newR << null
newRows << newR
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def index = 0
sqlInstance.withBatch(10000, sql){ ps->
for(def r : newRows) {
index++
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
index
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("\\|@\\|")
rows << row
}
return rows;
}
}
@@ -67,50 +67,50 @@ class BigdataDealBatch extends BaseBatch{
}
println "curDate:"+curDate
for(def city : citys){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
}
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
//
// saveToPgSql(rows)
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
saveToPgSql(rows)
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319, houseIdsfilter);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230318, 20230419, houseIdsfilter);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
@@ -146,7 +146,9 @@ class CcbCommonUtils {
public static def findGuidePrice(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
if(startDate && endDate){
q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
}
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
@@ -428,7 +430,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","Ãù´äÀ¶Íå")
// q.put("name","½õÄÏÆÖÈýÐ¡Çø")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["ĆŐͨץՏ","ÉĚץ"]))
@@ -660,4 +662,13 @@ class CcbCommonUtils {
pgsqlInstance.executeUpdate(sql2)
}
public static def findDistGovPrice(def city){
def ids = []
pgsqlInstance.eachRow("SELECT gujia_bds_id from base.t0831_community_guide_price_gj_h where urbn_nm='${city}'",{it->
ids << it.gujia_bds_id
});
return ids;
}
}
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-02-27", "2023-03-21")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-21")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//更新静态数据表
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202302//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202212
def endDate = 202302
def sDate= 202303//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202301
def endDate = 202303
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -46,6 +46,15 @@ class ThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
// def gocPrices = CcbCommonUtils.findDistGovPrice("ÉîÛÚ")
// def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val=202303 and gujia_bds_id=?"
// pgsqlInstance.withBatch(500, sql){ ps->
// for(def id : gocPrices){
// ps.addBatch(id)
// }
// }
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
@@ -53,13 +62,20 @@ class ThirdPartyPriceBatch extends BaseBatch {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
def gocIdsPrices = []
if(city == "ÉîÛÚ"){
gocIdsPrices = CcbCommonUtils.findDistGovPrice(city)
}
def statMap = getDistrictStatByCity(city, weekend)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def thirdPriceMap = findThirdPartyPrice(city, statMap)
//def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用)
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices)
// def rets = createData3(cityCodeDatas.get(city))//新第三方均价表,要做2步,麻烦
rows.addAll(rets)
println index+" "+city+" "+rets.size() +" "+rows.size()
@@ -187,10 +203,12 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){
def allRows = []
for(def d : dists){
if(gocPrices && gocPrices.contains(d._id.toString())){
continue;
}
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
@@ -66,10 +66,7 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
// exportThirdPartyPrice2(thirdPartyMonth, new_dists)
exportPgThirdPartyPrice2(thirdPartyMonth, new_dists)
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
// exportCommunityBasePrice(lastMonth)
@@ -78,16 +75,22 @@ class ExportDistrictRongHe extends BaseBatch{
// exportBigdataSupplierStatistics(lastMonth)
// exportBigdataModelingBase(lastMonth)
exportPgCommunityBasePrice(lastMonth)
exportPgCommunityPdescribeGj(lastMonth)
exportPgGuideStatistics(lastMonth)
exportPgBigdataSupplierStatistics(lastMonth)
exportPgBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
}
if(curDate.toString().endsWith("28")){
exportPgBigdataSupplierStatistics(lastMonth)
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
// exportThirdPartyPrice2(thirdPartyMonth, new_dists)
exportPgThirdPartyPrice2(thirdPartyMonth, new_dists)
exportPgCommunityBasePrice(lastMonth)
exportPgCommunityPdescribeGj(lastMonth)
exportPgBigdataModelingBase(lastMonth)
exportPgGuideStatistics(lastMonth)
}
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
@@ -50,6 +50,7 @@ class CityInfoAnaBatch extends BaseBatch {
final static DBCollection HOUSE_LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia", "house_loudong", MongoUtils.MongoDB_Domain_mongo);
final static DBCollection LOUDONG_CONN = MongoUtils.getCollectionByName("fangjia_base", "loudong", MongoUtils.MongoDB_Domain_mongo);
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000);
final static DBCollection FACILITY_CONN = MongoUtils.getCollectionByName("fangjia", "district_facilitys", MongoUtils.MongoDB_Domain_mongo)
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616')
final static DSCollection SEAWEED = DbUtils.seaweed
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -62,35 +63,41 @@ class CityInfoAnaBatch extends BaseBatch {
car_site :'carSite',estate_charge :'estateCharge',floorplan_pics :'floorplanPics',out_pics :'outPics',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',licence :'licence',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias'
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount'
]
public static final def OFFICE_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure',
facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics'
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure',
facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo',licence:'licence',
sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany',estate_charge:'estateCharge',
floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife',
conditioner_time:'conditionerTime',elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount',
goods_elevator_count:'goodsElevatorCount',network_way:'networkWay',fitment_level:'fitmentLevel',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea',
is_foreign:'foreign',is_register:'register',lighting_ventilation:'lightingVentilation',lobby_height:'lobbyHeight'
]
public static final def SHOP_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',
rent_type :'rentType',lease_term :'leaseTerm',car_site :'carSite',estate_charge :'estateCharge',divisible :'divisible',floor :'floor',
finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate',
bay_area :'bayArea', property_life :'propertyLife',unified_manage :'unifiedManage',licence :'licence',business_status :'businessStatus',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',bed_kind:'bedKind',floor_info:'floorInfo',car_site_count :'carSiteCount',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics'
]
public static CITY_GRADE_MAP = [1000: "一线" , 1500 : "新一线" , 2000 : "二线" , 3000:"三线" ,4000 : "四线" ,5000 : "五线"]
final static def COUNT_CITYS = ["滁州","黄山","马鞍山","芜湖","宿州","宣城","北京","白银","定西","甘南","金昌","兰州","临夏","陇南","平凉","庆阳","天水","武威","张掖","东莞","佛山","广州","惠州","江门","梅州","清远","深圳","玉林","安顺","贵阳","铜仁","三亚","海口","承德","邯郸","衡水","廊坊","秦皇岛","石家庄","开封","洛阳","大庆","鄂州","黄冈","十堰","武汉","宜昌","常德","长沙","常州","淮安","连云港","南京","南通","苏州","泰州","无锡","宿迁","徐州","盐城","扬州","镇江","景德镇","九江","上饶","鹰潭","抚顺","辽阳","盘锦","沈阳","铁岭","营口","呼和浩特","石嘴山","银川","海东","西宁","滨州","德州","济南","济宁","青岛","潍坊","淄博","太原","宝鸡","渭南","西安","咸阳","上海","成都","达州","德阳","乐山","凉山州","泸州","眉山","绵阳","南充","内江","遂宁","雅安","宜宾","资阳","自贡","天津","丽江","昭通","杭州","湖州","嘉兴","金华","丽水","宁波","衢州","绍兴","台州","温州","舟山","重庆"];
final static def DIST_FACILITY_LIST = ["bank", "hospital", "bus", "shop", "university", "middle_school", "primary_school", "kindergarten", "food_shop", "supermarket","atm","school","subway"]
/**
* 统计结果接收邮箱列表
*/
final static def receivers = [
"puxueli@fangjia.com",
"wangchencen@fangjia.com",
"gaojuan@fangjia.com",
"caoyuyan@fangjia.com",
"muliangshu@fangjia.com"
]
@@ -116,7 +123,9 @@ class CityInfoAnaBatch extends BaseBatch {
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
allCitys.each {k,v ->
CITYS << k.toString();
// if(["上海","深圳","北京","广州","杭州","苏州","重庆","成都"].contains(k.toString())){
CITYS << k.toString();
// }
}
// 所有城市的人口信息
@@ -146,7 +155,7 @@ class CityInfoAnaBatch extends BaseBatch {
allCityDisCount.put(it.name, it.count.intValue());
}
}
// 城市新盘数
def allCityNewDisCount = [:];
disQuery = new BasicDBObject();
@@ -162,7 +171,7 @@ class CityInfoAnaBatch extends BaseBatch {
allCityNewDisCount.put(it.name, it.count.intValue());
}
}
// 城市写字楼数
def allCityOfficeCount = [:];
disQuery = new BasicDBObject();
@@ -178,8 +187,8 @@ class CityInfoAnaBatch extends BaseBatch {
allCityOfficeCount.put(it.name, it.count.intValue());
}
}
// 城市商铺数
def allCityShopCount = [:];
disQuery = new BasicDBObject();
@@ -503,7 +512,7 @@ class CityInfoAnaBatch extends BaseBatch {
entity.put('official_population', curInfo.official_population);
entity.put('district_avg_population', curInfo.district_avg_population);
entity.put('rate', curInfo.rate);
removeCurInfo(cityName, weekend);
} else {
def cityHouseStat = allHouseStats.get(cityName);
@@ -513,13 +522,13 @@ class CityInfoAnaBatch extends BaseBatch {
} else if (cityHouseStat && cityHouseStat.standard_people_count) {
entity.put('official_population', cityHouseStat.standard_people_count);
}
if (lastInfo && lastInfo.district_avg_population) {
entity.put('district_avg_population', lastInfo.district_avg_population);
} else if (cityHouseStat && cityHouseStat.district_avg_population) {
entity.put('district_avg_population', cityHouseStat.district_avg_population);
}
if (lastInfo && lastInfo.rate) {
entity.put('rate', lastInfo.rate.toDouble());
} else if (cityHouseStat) {
@@ -565,6 +574,8 @@ class CityInfoAnaBatch extends BaseBatch {
entity.put("price_credit_gte_00_coverage", DataUtil.roundDouble(creditGte00Count/count, 4))
}
entity.put('district_detail', districtDetail(disCount, cityName));
entity.put('district_facilitys_detail', districtFacilitysDetail(disCount, cityName));
} else {
entity.put('district_detail', [:]);
}
@@ -1079,6 +1090,75 @@ class CityInfoAnaBatch extends BaseBatch {
districtDetail.put('rentPrice', roundDouble(pc, disCount));
return districtDetail;
}
def districtFacilitysDetail(def disCount, def cityName){
DBObject query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'district');
query.put('visible', 0);
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
def facilityMap = findFacilityByCity(cityName)
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def countMap = [:]
while(iter.hasNext()) {
def dbo = iter.next()
def facilityCountMap = facilityMap.get(dbo.city+"##"+dbo.region+"##"+dbo.name)
if(facilityCountMap){
DIST_FACILITY_LIST.each() {
def count = facilityCountMap.get(it)
if (count) {
if(countMap.containsKey(it)) {
def tempCount = countMap.get(it) + 1
countMap.put(it, tempCount)
}else {
countMap.put(it, 1)
}
}
}
}
}
def facilityDetail = [:];
DIST_FACILITY_LIST.each() {it->
if(countMap.containsKey(it)) {
facilityDetail.put(it, roundDouble(countMap.get(it), disCount))
}else {
facilityDetail.put(it, 0)
}
}
return facilityDetail;
}
def findFacilityByCity(city){
DBObject query = new BasicDBObject();
query.put('city', city);
DBCursor cur = FACILITY_CONN.find(query).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def distMap = [:]
while(iter.hasNext()) {
def dbo = iter.next()
def countMap = [:]
DIST_FACILITY_LIST.each {
def values = dbo.get(it)
def count = 0
if(values){
count = values.size()
countMap.put(it, count)
}
}
if(countMap){
distMap.put(dbo.city+"##"+dbo.region+"##"+dbo.district_name, countMap)
}
}
return distMap;
}
double roundDouble(int a, int base) {
if(a == 0 || base == 0) {
@@ -1177,7 +1257,7 @@ class CityInfoAnaBatch extends BaseBatch {
DBObject query = new BasicDBObject();
query.put('city', cityName);
query.put('cat', 'office');
query.put('visible', 0);
query.put('status', 0);
DBObject fields = new BasicDBObject()
OFFICE_FIELDS_MAP.each() {key,skey->
@@ -1187,7 +1267,7 @@ class CityInfoAnaBatch extends BaseBatch {
DSCursor cur = SEAWEED.find(query, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
def countMap = [:]
println cur.count()
while(iter.hasNext()) {
def dbo = iter.next()
OFFICE_FIELDS_MAP.each() {key,skey->
@@ -124,7 +124,7 @@ class ThirdPriceExportBatch extends BaseBatch{
add = true;
}
def tc_price = getLastThirdPrice(o.get("58tc_esf_price"), curPrice)
def tc_price = getLastThirdPrice(o.anjuke_esf_price, curPrice)//58tc_esf_price
row << tc_price
if(tc_price){
add = true;
@@ -0,0 +1,530 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.sql.Connection
import java.sql.DriverManager
import java.sql.PreparedStatement
import java.sql.Statement
import java.util.regex.Pattern
class OfficeProcess {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
def sqlInstance = Sql.newInstance("jdbc:clickhouse://192.168.0.216:8123/etl?max_memory_usage=50000000",
"goojia", "goojia123456", "ru.yandex.clickhouse.ClickHouseDriver")
static main(args) {
def filePath = "/home/goojia/gj/6城未匹配上的写字楼.xlsx"
if(CommonDatautils.isWindow()){
filePath = "D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx"
}
def task = new OfficeProcess();
// task.run("D:\\数据\\写字楼\\6城合并写字楼.xlsx")
// task.addPropertyType2("深圳")
// task.standardName(filePath)
// task.nameRepeatFiter(filePath)
// task.addressFiter(filePath)
// task.nameFiter(filePath)
// task.typeFiter(filePath)
// task.nameFlag(filePath)
// task.updateFlagToDB(filePath)
// task.updateFlagToDB2(filePath)
task.checkRegion()
}
def nameRepeatFiter(filePath){
def keys = []
ExcelUtils.modify(filePath, { row ->
if (!row['L'] && row['B']) {
def city = row['B']
def region = row['C']
def name = ""
if(row['K']){
name = row['K']
}else{
name = row['D']
}
def key = city+"##"+region+"##"+name
if(keys.contains(key)){
row['L'] = "名称去重"
println key
}else{
keys << key
}
}
})
}
def nameFiter(filePath){
ExcelUtils.modify(filePath, { row ->
def name = row['K']
if(!row['L'] && !name){
row['L'] = "名称过滤"
}
if(!row['L'] && name){
if(name.contains("公司")){
row['L'] = "名称过滤"
}
}
if(!row['L'] && name){
if(isStandardRoad(name)){
row['L'] = "名称过滤"
}
}
if(!row['L'] && name){
def fiters = ["厂房","测试", "服务","综合服务","酒店","深圳特专","工业","经济","产业","居委会","宿舍","购物","休闲","文化","训练","出租","浙菜","值班岗","招租","商住","办公室","配套","美食","博物馆","物流","商铺","公社","商城","值班室","暂停","创业社区","招商","宾馆","器材","教育","门诊","妇产科","商贸城","电子城","超市","零售"]
for(def key : fiters){
if(name.contains(key)){
row['L'] = "名称过滤"
break;
}
}
}
if(!row['L'] && name){
def fiters = ["大厦","裙楼",'花园','大楼','大堂','南塔','北京','大道','主楼','中国','中心','广场','鸟巢','小区','单位','办公','北塔'
,'写字楼','检验楼','办公楼','接待区','商务楼','文体楼','市中心','产业园','科技楼','行政楼','病房43、44、45','律己楼','调度楼','辉腾楼','总务楼'
,'7幢货梯','现厂B栋','(马连道南街)','中央电视台','后座','西大楼','B&C','商业用楼','财务中心','地铁','一瓶','外贸楼','业务楼','研发大楼','结算中心','马场','商务']
for(def key : fiters){
if(name == key){
row['L'] = "名称过滤"
break;
}
}
}
if(!row['L'] && row['J'] && name){
if(row['J'].toString().contains("商务写字楼") && name.contains("公寓")){
row['L'] = "名称过滤"
}
}
if(!row['L'] && name){
if(findPattern(name,'^[A-Za-z0-9]+$') && name.toString().length()<=3){
row['L'] = "名称过滤"
}else if(name.toString().length()<2){
row['L'] = "名称过滤"
}
}
if(!row['L'] && name){
def filters = ['筹备组','小区','家属房','家属院','家属楼','自编','市场','项目','批发城','传达室','梯','型材','学院','工地','冷冻厂','机关楼','阅读室','议事厅','电脑城','童装城','派出所','保安室','散盘','店','工作站',"物业楼",'物业','驿站','锅炉房','处理中心',"销售中心",'政府','馆','捐赠点','百货','工厂','工场','检测场','职工之家','移动','中心站','楼门','职场','鲜花基地','医院','装饰城','实训楼','业务楼','科研楼','学校','交易城','管理楼','考场','通道','团购','楼上','运营中心','商业城']
for(def key : filters){
if(name.toString().endsWith(key)){
row['L'] = "名称过滤"
break;
}
}
}
})
}
def typeFiter(filePath){
ExcelUtils.modify(filePath, { row ->
if(!row['L'] && row['J']){
def fiters = ["商住","服务","住宅小区","酒店","商业综合体","住宅底商"]
for(def key : fiters){
if(row['J'].toString().contains(key)){
row['L'] = "类型过滤"
break;
}
}
}
if(!row['L'] && row['B'] && !row['J']){
row['L'] = "类型空白过滤"
}
})
}
def nameFlag(filePath){
ExcelUtils.modify(filePath, { row ->
if(!row['L'] && row['D']) {
def fiters = ["大厦","大楼","商务楼","商务中心","广场","写字楼","办公楼","科技园","国际大厦","商务大厦","商务广场","创园","创意","中心","营销中心"]
for(def key : fiters){
if(row['D'].toString().contains(key)){
if(row['D']=="广场"){
}
row['L'] = "名称关键字[添加]"
break;
}
}
}
if(!row['L'] && row['J']){
if(row['J'].toString().contains("园区") || row['J'].toString().trim()=="纯写字楼" || row['J'].toString().trim()=="标准写字楼"){
row['L'] = "类型[添加]"
}
if(!row['L'] && row['J'].toString().trim()=="纯写字楼 标准写字楼" || row['J'].toString().trim()=="商务中心" || row['J'].toString().trim()=="商务中心 标准写字楼"){
row['L'] = "类型[添加]"
}
}
if(!row['L'] && row['J'] && row['D']){
if(row['J'].toString().contains("商务写字楼") && !row['D'].toString().contains("公寓")){
row['L'] = "类型[添加]"
}
}
if(!row['L'] && row['J'] && row['D']){
if(row['J'].toString().contains("纯写字楼")){
row['L'] = "类型[添加]"
}
}
})
}
def addressFiter(filePath){
def keyMap = [:]
ExcelUtils.modify(filePath, { row ->
if(!row['L'] && row['F']){
def city = row['B']
def region = row['C']
def name = ""
if(row['K']){
name = row['K']
}else{
name = row['D']
}
def address = row['F']
def key = city+"##"+region+"##"+address
def keys = keyMap.keySet()
if(keys.contains(key)){
def values = keyMap.get(key)
if(values){
for(def n : values){
def score = CosineSimilarAlgorithm.getSimilarityRatio(name, n)
if(score>=0.8){
row['L'] = "地址去重"
break;
}
println address +" "+ name+" "+n +" "+score
}
}
}else{
def values = keyMap.get(key)
if(!values){
values = []
keyMap.put(key, values)
}
values << name
}
}
});
}
def standardName(filePath){
def index = 0
ExcelUtils.modify(filePath, { row ->
index++
if(!row['K'] && row['D']){
def name = row['D']
def newName = name.replaceAll("-|·|(|)|//|\\?|“|”","").replaceAll("\\w+幢\\w+座","").replaceAll(" ","").replaceAll("\\.","").replaceAll("丨","")
newName = newName.replaceAll("-?\\w+号商业楼","").replaceAll("\\?","")
newName = newName.replaceAll("\\w+楼\\w+室","")
newName = newName.replaceAll("\\w+号\\w+(座|幢)","")
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十)号楼(一|二|三|四|五|六|七|八|九|十)单元","")
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|十一)?幢?(一|二|三|四|五|六|七|八|九|十|东|南|西|北)(塔楼|楼)","")
newName = newName.replaceAll("-?(第)?\\w+(号写字楼|号楼|号大楼|楼|座|栋|南楼|北楼|东楼|西楼|区|单元)","")
newName = newName.replaceAll("(一|二|三|四|五|六|七|八|九|十|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾)栋","")
newName = newName.replaceAll('(东|南|西|北)座$',"")
newName = newName.replaceAll("1期","一期").replaceAll("2期","二期").replaceAll("3期","三期").replaceAll("4期","四期").replaceAll("5期","五期").replaceAll("6期","六期").replaceAll("7期","七期").replaceAll("8期","八期").replaceAll("9期","九期").replaceAll("10期","十期").replaceAll("1-5期","一-五期").replaceAll("5-6期","六-七期")
newName = newName.toUpperCase().replaceAll("(装修中)","").replaceAll("\\(\\)","")
if(newName){
def md = DataUtil.match(newName, Pattern.compile("^[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中]{0,1}[写字|办公|楼|座|区|栋|组|幢|0-9|东|南|西|北|中|号]?[0-9|a-z|A-Z|一|二|三|四|五|六|七|八|九|十|零|壹|贰|叁|肆|伍|陆|柒|捌|玖|拾|东|南|西|北|中|办公|写字]?[楼|座|区|栋|组|幢|0-9|东|南|西|北|中]\$"))
if(md){
newName = ''
}
}
row['K'] = newName
}
})
}
def addPropertyType2(runCity){
def map = [:]
def datas = ExcelUtils.read("D:\\数据\\写字楼\\6城未匹配写字楼.xlsx",0,1)
if(datas){
for(def d : datas){
def key = d['B']+"##"+d['C']+"##"+d['D']
def list = map.get(key)
if(!list){
list = []
map.put(key, list)
}
if(d['M'] && !list.contains(d['M'])){
list << d['M']
}
if(d['U'] && !list.contains(d['U'])){
list << d['U']
}
}
}
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\6城未匹配上的写字楼.xlsx", { row ->
index++
def city = row['B']
def region = row['C']
def name = row['D']
def types = map.get(city+"##"+region+"##"+name)
if(types){
row['J'] = types.join(" ")
}
})
}
def addPropertyType(){
def index = 0
ExcelUtils.modify("D:\\数据\\写字楼\\未匹配写字楼.xlsx", { row ->
index++
def city = row['B']
def region = row['C']
def name = row['D']
def types = findDatasByDB(city, region, name)
if(types){
row['J'] = types.join(" ")
}
println index
})
}
def findDatasByDB(city, region, name){
def sql = "select * from office where city='${city}' and region='${region}' and name='${name}'"
def types = []
sqlInstance.eachRow(sql, {r->
if(r.property_type && !types.contains(r.property_type)){
types << r.property_type
}
})
return types
}
def run(filepath){
def index = 0
ExcelUtils.multiThreadingModify(filepath,{row->
index++
def city = row['B']
def region = row['C']
def name = row['D']
def address = row['I']
if(city ){
def jsonObject = OpenApiClient.addressMatch(city, name, "office", false, region, null)
// if(jsonObject && jsonObject.precise == -1){
// jsonObject = OpenApiClient.addressMatch(city, address, "office", false, region, name)
// }
if (jsonObject != null && jsonObject.precise != -1) {
row['J'] = jsonObject.get("id")
row['K'] = jsonObject.get("city")
row['L'] = jsonObject.get("district")
row['M'] = jsonObject.get("name")
row['N'] = jsonObject.get("credit")
row['O'] = jsonObject.get("propertyType")
}
}
println index
})
}
def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)?");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
if(s.toString().endsWith("路")||s.toString().endsWith("街")){
return true;
}
return false
}
def findPattern(def address, def regex) {
Pattern pattern = Pattern.compile(regex);
def matcher = pattern.matcher(address)
if (matcher.find()) {
return true
}
return false;
}
def updateFlagToDB(filePath){
def datas = ExcelUtils.read(filePath,0,1)
def total = datas.size()
println total
def index = 0
def rets = []
for(def d : datas){
def sql = "alter table office update option='${d['L']}', format_name='${d['K']}' where city='${d['B']}' and region='${d['C']}' and name='${d['D']}';"
index++
rets << sql
//sqlInstance.executeUpdate(sql)
//println index+"/"+ total
println sql
}
def filePath1 = "d:\\Data1.txt"
File file = new File(filePath1)
def s = new StringBuffer()
for(def r : rets){
s.append(r+"\n")
}
file.write(s.toString())
}
def updateOption(city, region, name, source, flag){
def sql = "alter table office update region_fence='${flag}' where city='${city}' and region='${region}' and name='${name}' and source='${source}';"
sqlInstance.executeUpdate(sql)
}
def batchUpdate(rets){
// 初始化驱动
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
// url
String url = "jdbc:clickhouse://192.168.0.216:8123/etl";
// 用户名密码
String user = "goojia";
String password = "goojia123456";
// 登录
Connection connection = DriverManager.getConnection(url, user, password);
Statement stmt = connection.createStatement();
def count = 0
def sql = "alter table office update region_fence=? where city=? and region=? and name=? and source=?;"
PreparedStatement pstmt = connection.prepareStatement(sql)
for(def d : rets){
count++
pstmt.setString(1, d.flag)
pstmt.setString(2, d.city)
pstmt.setString(3, d.region)
pstmt.setString(4, d.name)
pstmt.setString(5, d.source)
println count+" "+d.city+" "+d.name
pstmt.addBatch()
if(count % 1 ==0) {
try {
pstmt.executeBatch()
} catch (Exception e) {
e.printStackTrace()
}
}
}
pstmt.executeBatch()
}
def updateFlagToDB2(filePath){
// 初始化驱动
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
// url
String url = "jdbc:clickhouse://192.168.0.216:8123/etl";
// 用户名密码
String user = "goojia";
String password = "goojia123456";
// 登录
Connection connection = DriverManager.getConnection(url, user, password);
Statement stmt = connection.createStatement();
def count = 0
def datas = ExcelUtils.read(filePath,0,1)
def sql = "alter table office update option=?, format_name=? where city=? and region=? and name=?;"
PreparedStatement pstmt = connection.prepareStatement(sql)
for(def d : datas){
if(d['B']){
count++
pstmt.setString(1, d['L'])
pstmt.setString(2, d['K'])
pstmt.setString(3, d['B'])
pstmt.setString(4, d['C'])
pstmt.setString(5, d['D'])
println count+" "+d['L']+" "+d['K']+" "+d['B']+" "+d['C']+" "+d['D']
pstmt.addBatch()
if(count % 100 ==0) {
pstmt.executeBatch()
}
}
}
pstmt.executeBatch()
}
def checkRegion(){
def citys = ["北京","广州","重庆","天津","成都","南京"]//"上海","深圳",
for(def city : citys){
def regionPointsMap = findRegionRange(city)
def addDatas = findAddDatasByClickhouse(city)
println "需处理的区域围栏:"+addDatas.size()
def index = 0
def rets = []
for(def data : addDatas){
index ++
if(data.lng){
def flag = isInArea(data.lng.toString().toDouble(), data.lat.toString().toDouble(), regionPointsMap.get(data.region))
def mark = flag?'T':'F'
// rets << ["city":data.city, "region":data.region, "name":data.name, "source":data.source, "flag":mark]
updateOption(data.city, data.region, data.name, data.source, mark)
println index+" "+data.region + " "+data.name+" "+ flag
}
}
// batchUpdate(rets)
}
}
def findAddDatasByClickhouse(city){
def sql = "select city,region, name,option,lng,lat,source from office where city='${city}' and option like '%添加%'"
def datas = []
sqlInstance.eachRow(sql, {r->
datas << ["city":r.city,"region":r.region,"name":r.name, "lat":r.lat, "lng":r.lng, "option":r.option,"source":r.source]
})
return datas
}
def findRegionRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline)
}
return regionMap
}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
}
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
def esSources = ["ËÑ·¿":"ËÑ·¿"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
static main(args) {
@@ -43,7 +43,7 @@ class SynchDistrictPrice extends BaseBatch{
println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = ["Á¹É½ÖÝ"]//CommonUtils.findAllCity()
def citys = CommonUtils.findAllCity()
esSources.each{esSource, mgSource->
for(def city : citys){
def rets = findAllEsData(city, esSource, mgSource, mdate)
@@ -1,9 +1,6 @@
package com.fangjia.service.datastore;
import com.fangjia.core.util.BeanUtils;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.DateUtil;
import com.fangjia.core.util.HouseValidator;
import com.fangjia.core.util.*;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.SortParam;
import com.fangjia.dal.SrQuery;
@@ -51,7 +48,7 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{
private static String[] HOUSE_FIELDS = new String[]{"city", "region", "districtName", "totalPrice", "avgPrice",
"area", "fitment", "direction", "floor", "height", "publishDate", "address", "estateType"};
public static String[] TAGS =new String[]{"独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"};
private DealPriceDao dealPriceDao;
private HouseDao houseDao;
@@ -191,7 +188,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{
}
List<SampleWrapper> querySamples= new ArrayList<SampleWrapper>();
List<SampleWrapper> samples = findSampleByDistrict(district.getCity(), district.getRegion(), district.getName(), district.getAlias(), district.getMasterDistrictNname(), district.getEstateType2(),queryBean.getPropertyTypeCode(), queryBean.getUpdateDate(), districtStat);
scoreForSample(samples, target);
// scoreForSample(samples, target);
sortForSample(samples, target);
for(SampleWrapper sample : samples){
sample.setProperty(queryBean.getProperty());
}
@@ -205,7 +203,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{
for(SampleWrapper sample : nearlySample){
sample.setProperty(d);
}
scoreForSample(nearlySample, target);
// scoreForSample(nearlySample, target);
sortForSample(samples, target);
querySamples.addAll(nearlySample);
if(querySamples.size() > max){
break;
@@ -481,6 +480,56 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{
return samples;
}
private List<SampleWrapper> sortForSample(List<SampleWrapper> samples, final SampleWrapper targetSample){
if(samples.isEmpty()){
return samples;
}
// if(samples!=null && samples.size() < max){
// return samples;
// }
Collections.sort(samples, new Comparator<SampleWrapper>() {
public int compare(SampleWrapper o1, SampleWrapper o2) {
Double d1 = Math.abs(o1.getStrctureArea() - targetSample.getStrctureArea());
Double d2 = Math.abs(o2.getStrctureArea() - targetSample.getStrctureArea());
// return (int) (d1 - d2);
return d1.compareTo(d2);
}
});
List<Integer> prices = new ArrayList<Integer>();
for(SampleWrapper sample : samples){
if(!prices.contains(sample.getUnitPrice())){
prices.add(sample.getUnitPrice());
if(prices.size()>=10){
break;
}
}
}
final Integer midPrice = (int)median(prices);
Collections.sort(samples, new Comparator<SampleWrapper>() {
public int compare(SampleWrapper o1, SampleWrapper o2) {
Integer d1 = Math.abs(o1.getUnitPrice() - midPrice);
Integer d2 = Math.abs(o2.getUnitPrice() - midPrice);
return d1.compareTo(d2);
}
});
return samples;
}
private static double median(List<Integer> total) {
double j = 0;
Collections.sort(total);
int size = total.size();
if(size % 2 == 1){
j = total.get((size-1)/2);
}else {
j = (total.get(size/2-1) + total.get(size/2) + 0.0)/2;
}
return j;
}
/**
* ²éÕÒÐ¡ÇøÑù±¾
* @param city
@@ -514,8 +563,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{
houseQuery.add(new QueryParam("region", region ));
if(districtStat != null && districtStat.getAvgPrice() > 0){
int avgPrice = districtStat.getAvgPrice();
houseQuery.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_EQUAL_OPERATOR, (int)(avgPrice * 0.85 )));
houseQuery.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_EQUAL_OPERATOR, (int)(avgPrice * 1.15 )));
houseQuery.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_EQUAL_OPERATOR, (int)(avgPrice * 0.5 )));//0.85
houseQuery.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_EQUAL_OPERATOR, (int)(avgPrice * 1.5 )));//1.15
}
BooleanQuery nameQuery = new BooleanQuery();
int c = 0;
@@ -548,6 +597,21 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{
int maxFloor = 0;
boolean allSampleHasHeight = true;
for(House house : houses){
if(StringUtils.isNotBlank(estateType) && estateType.equals("01")){//01:普通住宅 02:别墅
if(StringUtils.isNotBlank(house.getEstateType()) && house.getEstateType().contains("别墅")){
continue;
}
if(isVilla(house.getArea(), house.getName(), house.getHeight(), house.getRoom())){
continue;
}
}else if(StringUtils.isNotBlank(estateType) && estateType.equals("02")){
if(house.getHeight()!=null && house.getHeight()>=10){
continue;
}
if(house.getFloor()!=null && house.getFloor()>=10){
continue;
}
}
SampleWrapper sample = convertHouse2Sample(house);
if(isAvailableSample(sample)){
@@ -859,6 +923,39 @@ public class CzbankSampleServiceImpl implements CzbankSampleService{
return null;
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(Double area, String name, Integer height, Integer room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
public void setDealPriceDao(DealPriceDao dealPriceDao) {
this.dealPriceDao = dealPriceDao;
}