Merge remote-tracking branch 'origin/master'
This commit is contained in:
+5
-2
@@ -435,8 +435,11 @@ public class AddrSvcController {
|
||||
if (json.containsKey("result")) {
|
||||
//我们的结果
|
||||
JSONObject result = json.getJSONObject("result");
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
// 如果是无效信息,则用我们的
|
||||
if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) {
|
||||
basicModel.setResult(result);
|
||||
}//我们省级及以上有开发区,直接用我们的结果
|
||||
else if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else {
|
||||
|
||||
@@ -55,38 +55,44 @@ public class Controller {
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 缓存刷新接口
|
||||
* @return
|
||||
*/
|
||||
@Authority(role = Role.NONE)
|
||||
public Object refreshCacheAction(RefreshCacheModel model){
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
Token token = FangjiaApiUtil.parse2Token(model.getToken());
|
||||
if(token == null){
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
|
||||
return mav;
|
||||
}
|
||||
final ThreadLocalData threadLocalData = ThreadLocalData.getInstance();
|
||||
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
|
||||
// 在验证appKey
|
||||
ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId());
|
||||
if(apiUser == null){
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
|
||||
return mav;
|
||||
}
|
||||
try {
|
||||
apiUserAuthorityService.findByUserId(apiUser.getId());
|
||||
if(StringUtils.isNotBlank(model.getViewConfigUrl())){
|
||||
apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl());
|
||||
}
|
||||
} catch (IllegalAccessException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功"));
|
||||
return mav;
|
||||
}
|
||||
/**
|
||||
* 缓存刷新接口
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
@Authority(role = Role.NONE)
|
||||
public Object refreshCacheAction(RefreshCacheModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
Token token = FangjiaApiUtil.parse2Token(model.getToken());
|
||||
if (token == null) {
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
|
||||
return mav;
|
||||
}
|
||||
final ThreadLocalData threadLocalData = ThreadLocalData.getInstance();
|
||||
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
|
||||
// 在验证appKey
|
||||
ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId());
|
||||
if (apiUser == null) {
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
|
||||
return mav;
|
||||
}
|
||||
try {
|
||||
// 获取用户
|
||||
User user = userService.findById(apiUser.getId());
|
||||
if (null != user && StringUtils.isNotBlank(user.getUserName())) {
|
||||
userService.findByUserName(user.getUserName());
|
||||
}
|
||||
apiUserAuthorityService.findByUserId(apiUser.getId());
|
||||
if (StringUtils.isNotBlank(model.getViewConfigUrl())) {
|
||||
apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl());
|
||||
}
|
||||
} catch (IllegalAccessException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+8
-8
@@ -95,10 +95,12 @@ public class GdBankController {
|
||||
} else if (("否").equals(result.get("countyLevelCity"))) {
|
||||
result.put("prefectureLevelCity", "是");
|
||||
}
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
// 如果是无效信息,则用我们的
|
||||
if (result.containsKey("addrVerify") && !"地址有效".equals(result.getString("addrVerify"))) {
|
||||
basicModel.setResult(result);
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
} else if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else {
|
||||
try {
|
||||
//如果是江苏省或者浙江省 调用齐梁的结果
|
||||
@@ -112,10 +114,9 @@ public class GdBankController {
|
||||
// 涉农结果不一样
|
||||
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
|
||||
//有市辖建制镇, 或者城市不一样 用我们
|
||||
if (result.containsKey("city")) {
|
||||
if ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
if (result.containsKey("city") && ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city"))
|
||||
&& StringUtils.isNotBlank(result.getString("city"))))) {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
if (("是").equals(result.get("countyLevelCity"))) {
|
||||
map.put("prefectureLevelCity", "否");
|
||||
@@ -129,7 +130,6 @@ public class GdBankController {
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
// 分期分区与主小区映射关系表
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class FindStageDistrictBatch extends BaseBatch {
|
||||
|
||||
|
||||
+7
-1
@@ -34,7 +34,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -60, "yyyyMM"));
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
@@ -68,6 +68,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
@@ -679,6 +681,10 @@ class BigdataDealBatch extends BaseBatch{
|
||||
return map
|
||||
}
|
||||
|
||||
def updateCrawlTradeDate(){
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
|
||||
-4
@@ -678,10 +678,6 @@ class CcbCommonUtils {
|
||||
}
|
||||
|
||||
public static def deleteGuidedPrice(def dealdate){
|
||||
def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
|
||||
|
||||
def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'"
|
||||
pgsqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
|
||||
+1
-1
@@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240508--人工指导价.xls"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\5月国信达价格录入.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['J']){
|
||||
|
||||
+19
@@ -1890,6 +1890,25 @@ class CommonDatautils {
|
||||
def dbo = null;
|
||||
while(cur.hasNext()) {
|
||||
dbo = cur.next()
|
||||
if(dbo.value_type == 1){
|
||||
dbo.cal_type = "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
dbo.cal_type = "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
dbo.cal_type = "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
dbo.cal_type = "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
dbo.cal_type = "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
dbo.cal_type = "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
dbo.cal_type = "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
dbo.cal_type = "无方式"
|
||||
}else{
|
||||
dbo.cal_type = ""
|
||||
}
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
|
||||
+2
-2
@@ -22,8 +22,8 @@ class FzDistrictMatch {
|
||||
// task.addPrice()
|
||||
// task.addThirdPrice()
|
||||
|
||||
// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
|
||||
task.longyanPrice()
|
||||
task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
|
||||
// task.longyanPrice()
|
||||
}
|
||||
|
||||
def longyanPrice(){
|
||||
|
||||
+143
-48
@@ -30,6 +30,7 @@ class DevProcess {
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
// task.devSplitRoad()
|
||||
task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
|
||||
@@ -37,14 +38,17 @@ class DevProcess {
|
||||
// task.updateCityAndDist()
|
||||
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
|
||||
// task.addDevAddres2()//添加相差2公里范围内的地址
|
||||
// task.matchDevAndAddAddress()
|
||||
|
||||
// task.stat()
|
||||
// task.addNewDev()
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
task.synchDevTo43()
|
||||
// task.synchDevTo43()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def parseCityAndDist(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
|
||||
if(!d['J']){
|
||||
@@ -159,54 +163,111 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def devSplitRoad(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
|
||||
/*if(d['C'] && !d['V']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['V'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['W'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['X'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['Y'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['Z'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
}*/
|
||||
if(d['D'] && !d['M'] && d['P'] && d['P'].toString().toDouble().toInteger()==1){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['D'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['J'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['K'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['L'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['M'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['N'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
def i = 10;
|
||||
while(i<11){
|
||||
def fileName = "/home/goojia/gj/company-${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\company-${i}.xlsx"
|
||||
}
|
||||
},0)
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
/*if(d['C'] && !d['V']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['V'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['W'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['X'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['Y'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['Z'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
}*/
|
||||
if(d['C'] && d['B']!='河北省'){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C'])
|
||||
if(jsonObject){
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['J'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['K'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['L'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['M'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['N'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
|
||||
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
|
||||
d['O'] = jsonObject.get("dev")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def devNameSplitRoad(){
|
||||
def fileName = "/home/goojia/gj/company.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx"
|
||||
}
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['P'] && d['P']=="路正常" && !d['O']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||
if(jsonObject){
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['J'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['K'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['L'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['M'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['N'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
|
||||
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
|
||||
d['O'] = jsonObject.get("dev")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addFlag(){
|
||||
@@ -495,4 +556,38 @@ class DevProcess {
|
||||
}
|
||||
DEVELOPMENT_ZONE_43_COLL.insert(rets)
|
||||
}
|
||||
|
||||
|
||||
def matchDevAndAddAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区.xlsx",{d->
|
||||
if(d['P'] && d['P']=="路正常" && d['O']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['O'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
def address = d['M']+d['N']
|
||||
q.put("address", address)
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q)
|
||||
if(!sameObj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", obj.province)
|
||||
if(obj.city){
|
||||
udata.put("city", obj.city)
|
||||
}
|
||||
if(obj.district){
|
||||
udata.put("district", obj.district)
|
||||
}
|
||||
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "地址")
|
||||
udata.put("address", address)
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ class DistrictAliasProcess {
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\名称在映射别名中的小区(2).xlsx",{ d->
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d->
|
||||
if(d['H']=="是"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
|
||||
+49
@@ -10,6 +10,7 @@ import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.client.AggregateIterable
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
@@ -35,6 +36,7 @@ class TownMiddleProcess {
|
||||
static main(args) {
|
||||
def task = new TownMiddleProcess();
|
||||
task.run()
|
||||
|
||||
}
|
||||
def run(){
|
||||
//创建省份
|
||||
@@ -52,6 +54,13 @@ class TownMiddleProcess {
|
||||
//添加地级市标志
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
//打标记,将开发区与民政部相同的打上1
|
||||
// addTjjDevFlag()
|
||||
//打标记,用于标识乡镇街道是否输出及格式化输出街道
|
||||
// addStreetFlag()
|
||||
|
||||
// addDistrictDiffFlag()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -751,4 +760,44 @@ class TownMiddleProcess {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def addTjjDevFlag(){
|
||||
def q = new BasicDBObject("type", "区域")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||
for(def d : dbcursor){
|
||||
if(d.district && d.mzb_district_name && d.district==d.mzb_district_name){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("district_diff", 1) //民政部,统计局 名称相同 district_diff打1
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addStreetFlag(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道-结果.xlsx",{ row->
|
||||
if(row['I']){
|
||||
if(row['I']=="标0"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_output", 0) //0:不输出 1:输出
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
}else if(row['I']=="标1"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_output", 1) //0:不输出 1:输出
|
||||
if(row['G']){
|
||||
udata.put("street", row['G']) //街道中去除区域
|
||||
}
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addDistrictDiffFlag(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\区域-结果.xlsx",{ row->
|
||||
def flag = row['F'].toString().toDouble().toInteger()
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("district_diff", flag)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -354,13 +354,13 @@ class FangjiaClientUtils {
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
// println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
|
||||
+54
-87
@@ -48,11 +48,9 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataDealBatch(), args);
|
||||
@@ -61,59 +59,50 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
println "历史id加载量:"+dealIds.size()
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
// println "历史id加载量:"+dealIds.size()
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
//// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
//
|
||||
//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
//// saveToPgSql2(datas)
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
// saveToPgSql2(datas)
|
||||
}
|
||||
|
||||
def readTxt(def fileName){
|
||||
@@ -186,19 +175,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def getCommunityPrices(def city){
|
||||
def sql = "select commid, avg_price from community_base_price_gj where year_month=${endDate} and cityname='${city}'";
|
||||
def map = [:]
|
||||
try{
|
||||
sqlInstance.eachRow(sql,{
|
||||
map.put(it.commid, it.avg_price)
|
||||
});
|
||||
}catch(Exception e){
|
||||
getPpeGTE15Datas(city)
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
def findThirdPartyPrice(def city) {
|
||||
def q = new BasicDBObject()
|
||||
@@ -529,7 +506,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all")
|
||||
println city +" 小区量:"+dists.size()
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
@@ -537,9 +514,13 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -651,7 +632,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def findHistoryDealIds(def city){
|
||||
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'"
|
||||
def sql = "SELECT gujia_bds_mdl_id from base.bigdata_deal where urbn_nm='${city}'"
|
||||
def ids = []
|
||||
try{
|
||||
pgsqlInstance.eachRow(sql,{
|
||||
@@ -664,20 +645,6 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
return ids
|
||||
}
|
||||
|
||||
def getPpeGTE15Datas(def city){
|
||||
def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='成交' and pe>=15 and pe<=20 " +
|
||||
"and cityname='"+city+"'";
|
||||
def map = [:]
|
||||
try{
|
||||
sqlInstance.eachRow(sql,{
|
||||
map.put(it.commid, it.unit_price)
|
||||
});
|
||||
}catch(Exception e){
|
||||
getPpeGTE15Datas(city)
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
@@ -743,8 +710,8 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def delPgData(){
|
||||
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
pgsqlInstance.execute("delete from base.t0831_bigdata_deal_gj_h where p9_start_date='${month}'")
|
||||
println "base.t0831_bigdata_deal_gj_h 数据已删除"
|
||||
pgsqlInstance.execute("delete from base.bigdata_deal where p9_start_date='${month}'")
|
||||
println "base.bigdata_deal 数据已删除"
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
@@ -764,7 +731,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
def sql = "insert into base.bigdata_deal values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import org.bson.types.ObjectId
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 人工指导价同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
|
||||
|
||||
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwGuiDingPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
}
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def findFangDatas(def districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println " 小区量:"+districts.size()
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def obj = findMatchThirdPrice(dist)
|
||||
if(obj && obj.avg_price){
|
||||
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return result
|
||||
}
|
||||
|
||||
def findPersonDatas(def districts, def dataMap){
|
||||
def result = []
|
||||
for(def i=0;i<districts.size();i++) {
|
||||
def dist = districts[i];
|
||||
def price = dataMap.get(dist._id.toString())
|
||||
if (price) {
|
||||
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
|
||||
println dist.city + " " + dist.region + " " + dist.name + " " + price
|
||||
}
|
||||
}
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
def findMatchThirdPrice(dist){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", dist.city)
|
||||
q.put("match_region", dist.region)
|
||||
q.put("match_name", dist.name)
|
||||
q.put("s_date", sDate)
|
||||
q.put("source","搜房")
|
||||
def obj = dataProcessColl.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
if(areaData){
|
||||
ret << ps._id.toString()
|
||||
ret << dist._id.toString()
|
||||
ret << dist.name
|
||||
ret << cityCodeDatas.get(dist.city)
|
||||
ret << CcbCommonUtils.getCcbCity(dist.city)
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.guiding_value
|
||||
ret << "房价网"
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createDatas(cityCodeDatas, distCodeMap, datas){
|
||||
def rets = []
|
||||
if(datas) {
|
||||
for (def ps : datas) {
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(ps.region)
|
||||
if (areaData) {
|
||||
ret << ps.row_id
|
||||
ret << ps.dist_id
|
||||
ret << ps.name
|
||||
ret << cityCodeDatas.get(ps.city)
|
||||
ret << ps.city
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.avg_price
|
||||
ret << "房价网"
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
|
||||
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], "all");
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.guided_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(300, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+14
@@ -21,5 +21,19 @@ class FjwPgsqlCommUtils {
|
||||
return set
|
||||
}
|
||||
|
||||
public static def findDistGovPrice(def city){
|
||||
def ids = []
|
||||
sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it->
|
||||
ids << it.gujia_bds_id
|
||||
});
|
||||
return ids;
|
||||
}
|
||||
|
||||
|
||||
public static def deleteGuidedPrice(def dealdate){
|
||||
def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'"
|
||||
sqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+674
@@ -0,0 +1,674 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 第三方均价数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304//
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwThirdPartyPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = ["乌鲁木齐"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def stop = false
|
||||
def month = -1
|
||||
while(!stop){
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
println weekend.toString() +" "+sDate
|
||||
runDatas(citys)
|
||||
month = month -1
|
||||
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
|
||||
if(sDate == 201912){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def runDatas(citys){
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
|
||||
def gocIdsPrices = []
|
||||
if(city == "深圳"){
|
||||
gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city)
|
||||
}
|
||||
|
||||
def statMap = getDistrictStatByCity(city, weekend)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all")
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
|
||||
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices)
|
||||
rows.addAll(rets)
|
||||
println index+" "+city+" "+rets.size() +" "+rows.size()
|
||||
}
|
||||
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
if(data){
|
||||
def ret = []
|
||||
ret << third.id.toString()
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
ret << cityCodeDatas.get(d.city)
|
||||
ret << d.city
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
ret << sDate
|
||||
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
ret << third.zhugezhaofang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
ret << third.anjuke_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
ret << third.soufang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.lianjia_esf_price){
|
||||
ret << third.lianjia_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.cityhouse_esf_price){
|
||||
ret << third.cityhouse_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.maitian_esf_price){
|
||||
ret << third.maitian_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.beike_esf_price){
|
||||
ret << third.beike_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.get("365house_esf_price")){
|
||||
ret << third.get("365house_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.zhongyuan_esf_price){
|
||||
ret << third.zhongyuan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.tuboshi_esf_price){
|
||||
ret << third.tuboshi_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
ret << third.qfang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangtan_esf_price){
|
||||
ret << third.fangtan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangduoduo_esf_price){
|
||||
ret << third.fangduoduo_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
ret << third.get("5i5j_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){
|
||||
def allRows = []
|
||||
for(def d : dists){
|
||||
if(gocPrices && gocPrices.contains(d._id.toString())){
|
||||
continue;
|
||||
}
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
if(data){
|
||||
def dataMap = [:]
|
||||
dataMap.COMMID = d._id.toString()
|
||||
dataMap.COMMNAME = d.name
|
||||
dataMap.CITYID = cityCodeDatas.get(d.city)
|
||||
dataMap.CITYNAME = d.city
|
||||
dataMap.DISTID = data.district_code
|
||||
dataMap.DISTNAME = data.district
|
||||
dataMap.YEAR_MONTH = sDate
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
price_map.zhuge_avg_price = third.zhugezhaofang_esf_price
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
price_map.anjuke_avg_price = third.anjuke_esf_price
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
price_map.fang_avg_price = third.soufang_esf_price
|
||||
}
|
||||
if(third.lianjia_esf_price){
|
||||
price_map.lianjia_avg_price = third.lianjia_esf_price
|
||||
}
|
||||
if(third.cityhouse_esf_price){
|
||||
price_map.cityhouse_avg_price = third.cityhouse_esf_price
|
||||
}
|
||||
if(third.maitian_esf_price){
|
||||
price_map.maitian_avg_price = third.maitian_esf_price
|
||||
}
|
||||
if(third.beike_esf_price){
|
||||
price_map.ke_avg_price = third.beike_esf_price
|
||||
}
|
||||
if(third.get("365house_esf_price")){
|
||||
price_map.taofang_avg_price = third.get("365house_esf_price")
|
||||
}
|
||||
if(third.zhongyuan_esf_price){
|
||||
price_map.zhongyuan_avg_price = third.zhongyuan_esf_price
|
||||
}
|
||||
if(third.tuboshi_esf_price){
|
||||
price_map.tbs_avg_price = third.tuboshi_esf_price
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
price_map.qfang_avg_price = third.qfang_esf_price
|
||||
}
|
||||
if(third.fangtan_esf_price){
|
||||
price_map.fangtan_avg_price = third.fangtan_esf_price
|
||||
}
|
||||
if(third.fangduoduo_esf_price){
|
||||
price_map.fangduoduo_avg_price = third.fangduoduo_esf_price
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) {
|
||||
def rets = []
|
||||
for (def d : dists) {
|
||||
|
||||
def third = thirdPriceMap.get(d.region + "##" + d.name)
|
||||
if (third) {
|
||||
def data = distCodeMap.get(d.region)
|
||||
if (data) {
|
||||
third.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << cityCodeDatas.get(d.city)
|
||||
ret << data.district_code
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
def web = ""
|
||||
if(k=="zhugezhaofang_esf_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_esf_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="soufang_esf_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_esf_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_esf_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_esf_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="beike_esf_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="365house_esf_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_esf_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tuboshi_esf_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_esf_price"){
|
||||
ret << "Q房"
|
||||
}else if(k=="fangtan_esf_price"){
|
||||
ret << "房探"
|
||||
}else if(k=="fangduoduo_esf_price"){
|
||||
ret << "房多多"
|
||||
}else if(k=="5i5j_esf_price"){
|
||||
ret << "我爱我家"
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << sDate
|
||||
ret << v
|
||||
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData3(cityid){
|
||||
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def dataMap = [:]
|
||||
|
||||
dataMap.COMMID = r.COMMID
|
||||
dataMap.COMMNAME = r.COMMNAME
|
||||
dataMap.CITYID = r.CITYID
|
||||
dataMap.CITYNAME = r.CITYNAME
|
||||
dataMap.DISTID = r.DISTID
|
||||
dataMap.DISTNAME = r.DISTNAME
|
||||
dataMap.YEAR_MONTH = r.YEAR_MONTH
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(r.ZHUGE_AVG_PRICE){
|
||||
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ANJUKE_AVG_PRICE){
|
||||
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANG_AVG_PRICE){
|
||||
price_map.fang_avg_price = r.FANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.LIANJIA_AVG_PRICE){
|
||||
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.CITYHOUSE_AVG_PRICE){
|
||||
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.MAITIAN_AVG_PRICE){
|
||||
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.KE_AVG_PRICE){
|
||||
price_map.ke_avg_price = r.KE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TAOFANG_AVG_PRICE){
|
||||
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ZHONGYUAN_AVG_PRICE){
|
||||
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TBS_AVG_PRICE){
|
||||
price_map.tbs_avg_price = r.TBS_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.QFANG_AVG_PRICE){
|
||||
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGTAN_AVG_PRICE){
|
||||
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGDUODUO_AVG_PRICE){
|
||||
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.WIWJ_AVG_PRICE){
|
||||
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
|
||||
}
|
||||
dataMap.updatetime = r.UPDATETIME
|
||||
|
||||
allRows << dataMap
|
||||
});
|
||||
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
dataMap.put("id", db._id.toString())
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
dataMap.put("soufang_esf_price", db.soufang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
dataMap.put("qfang_esf_price", db.qfang_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
}
|
||||
|
||||
def resultMap = [:]
|
||||
map.each {key, value->
|
||||
if(value && value.size()>1){
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
|
||||
def price = statMap.get(key)
|
||||
if(price){
|
||||
def priceMap = [:]
|
||||
def startPrice = (int)(price - price*0.3)
|
||||
def endPrice = (int)(price*0.3 + price)
|
||||
value.each{pk,pv->
|
||||
if(pk.contains("price")){
|
||||
if(pv>=startPrice && pv<=endPrice){
|
||||
priceMap.put(pk,pv)
|
||||
}
|
||||
}else{
|
||||
priceMap.put(pk,pv)
|
||||
}
|
||||
}
|
||||
if(priceMap.size()>1){
|
||||
resultMap.put(key, priceMap)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
resultMap.put(key, value)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return resultMap
|
||||
}
|
||||
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"##"+d.district_name, d.avg_price)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+16
-15
@@ -33,11 +33,11 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\补充154价格.xlsx"
|
||||
def filePath = "D:\\Êý¾Ý\\½¨ÐÐ\\¶Ô±ê²¹³äÊý¾Ý.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
addFjwPrice(filePath)
|
||||
addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
|
||||
@@ -190,11 +190,11 @@ class AddPriceTest {
|
||||
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['M'])
|
||||
if (d['G']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(dist){
|
||||
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
|
||||
d['AN'] = thirdPrice
|
||||
d['Z'] = thirdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -206,7 +206,7 @@ class AddPriceTest {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
|
||||
q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def price = null
|
||||
@@ -216,10 +216,10 @@ class AddPriceTest {
|
||||
break;
|
||||
}
|
||||
|
||||
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
|
||||
price = db.beike_esf_price
|
||||
break
|
||||
}
|
||||
// if(!price && db.beike_esf_price && db.beike_esf_price>2000){
|
||||
// price = db.beike_esf_price
|
||||
// break
|
||||
// }
|
||||
}
|
||||
return price
|
||||
}
|
||||
@@ -256,14 +256,15 @@ class AddPriceTest {
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['F']){
|
||||
if(d['E'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(!d['W']){
|
||||
if(d['G'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(dist){
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['F'] = dbo.avg_price
|
||||
d['W'] = dbo.avg_price
|
||||
d['X'] = dbo.cal_type
|
||||
// d['J'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.*
|
||||
public class DbUtils{
|
||||
|
||||
public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com";
|
||||
public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
|
||||
//public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
|
||||
|
||||
public static final String COMMON_DB = "fangjia";
|
||||
public static final int COMMON_PORT = 27017;
|
||||
public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
|
||||
//public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
|
||||
public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||
|
||||
Reference in New Issue
Block a user