Merge remote-tracking branch 'origin/script_gj' into script_gj

This commit is contained in:
Jian Shen
2022-03-11 19:45:42 +08:00
19 changed files with 881 additions and 355 deletions
@@ -102,6 +102,8 @@ public class FjtEvaluation implements SqlEntity{
private String finishDateStr;//竣工时间
private String useType;//房屋用途
private Date downPdfDate;//下载报告时间
@GeneratedValue(strategy = GenerationType.AUTO)
@Id
@@ -750,6 +752,16 @@ public class FjtEvaluation implements SqlEntity{
this.useType = useType;
}
@Column(name = "down_pdf_date")
public Date getDownPdfDate() {
return downPdfDate;
}
public void setDownPdfDate(Date downPdfDate) {
this.downPdfDate = downPdfDate;
}
@Override
public String toString() {
return "FjtEvaluation{" +
@@ -38,6 +38,7 @@ public class OpenConfig extends Configuration {
private final String ipdatfile;
private final String baiduAk;
private final String defaultCity;
private final boolean isOpenAllAuthority;
@@ -54,6 +55,7 @@ public class OpenConfig extends Configuration {
baiduAk = configuration.get("fangjia.baidu.ak");
isOpenAllAuthority = configuration.getBoolean("fangjia.isOpenAllAuthority", false);
enabledAddressMatch = configuration.getBoolean("fangjia.addressMatch.enabled",false);
defaultCity = configuration.get("fangjia.default.city");
}
public static OpenConfig getConfig() {
@@ -134,4 +136,7 @@ public class OpenConfig extends Configuration {
return enabledAddressMatch;
}
public String getDefaultCity() {
return defaultCity;
}
}
@@ -89,6 +89,7 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService;
import com.fangjia.service.open.WaitCrawlKeywordsService;
import com.fangjia.service.sql.FjtEvaluationService;
import com.fangjia.service.web.AddressService;
import org.sothis.web.mvc.annotation.Param;
@Authority(role = Role.USER)
public class AddressController {
@@ -106,6 +107,7 @@ public class AddressController {
private HouseLoudongService houseLoudongService;
private FjtEvaluationService fjtEvaluationService;
private AddressMatchProxyService addressMatchProxyService;
private final static String AK = OpenConfig.getConfig().getBaiduAk();
private final static Logger LOGGER = LoggerFactory.getLogger(AddressController.class);
@@ -117,6 +119,8 @@ public class AddressController {
private static final String GRADE_NATION = "全国评分";
private static final String GRADE_CONSUME = "消费评分";
public final static OpenConfig CONFIG = OpenConfig.getConfig();
/**
* @deprecated see parse2Action
* @param model
@@ -1705,6 +1709,183 @@ public class AddressController {
return mav;
}
/**
* 地址标准化接口V2
*
* @param model
* @return
*/
public Object standard2Action(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 五级地址标准化接口
*
* @param model
* @return
*/
public Object fiveDivisionStructureAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
// 地址验证无效也输出标准化结果
params.put("flag", "true");
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 详细地址标准化接口
*
* @param model
* @return
*/
public Object divisionStructureAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
// 地址验证无效也输出标准化结果
params.put("flag", "true");
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 地址标准化专用测试接口
* @param model
* @return
*/
public Object structureForTestAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
// 地址验证无效也输出标准化结果
params.put("flag", "true");
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureForTest", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureForTest", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
public Object similarityAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
@@ -2363,7 +2363,7 @@ public class PropertyController {
mav.setModel(BasicModel.buildBadRequestModel("size(面积)不能为空"));
return mav;
}
District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(),
District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName().replace(" ",""),
"district");
if (null == district) {
mav.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
@@ -462,8 +462,14 @@ public class WeiXinEvaluationController {
}else if(bhDists.size()>1){
int mod = Integer.MAX_VALUE;
for(JSONObject jo : bhDists){
double price = Double.valueOf((String)jo.get("price"));
int bhprice = (int)price;
int bhprice = 0;
if (jo.get("price") == null || jo.get("price").equals("null")) {
bhprice = 0;
} else {
double price = Double.valueOf((String)jo.get("price"));
bhprice = (int)price;
}
int bhmod = Math.abs(bhprice - avgPrice);
if(bhmod<mod){
mod = bhmod;
@@ -1025,6 +1031,7 @@ public class WeiXinEvaluationController {
String reporNum = "147698"+ String.valueOf(evaluation.getId());
evaluation.setReportNum(reporNum);
evaluation.setDownPdfDate(new Date());
fjtEvaluationService.updateEvaluation(evaluation);
try {
@@ -441,7 +441,7 @@ public class BHBankUtils {
Map params = new HashMap();
params.put("queryId",queryId);
if(StringUtils.isNotBlank(reportMsg)){
params.put("reportMsg",reportMsg);
params.put("reportMsg",reportMsg.replace(" ",""));
}
@@ -100,4 +100,5 @@ fangjia.jms.email.queue=email.queue
fangjia.redis_nodes=192.168.0.79:6379,192.168.0.79:6380,192.168.0.79:6381,192.168.0.81:6382,192.168.0.81:6383,192.168.0.81:6384
fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2
fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2
fangjia.default.city = \u91CD\u5E86\u5E02
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlrewrite>
<global-params>
<param value="st" name="sort" type="attach"/>
<param value="pn" name="pager.currentPage" type="attach"/>
<!-- <param value="st" name="sort" type="attach"/>-->
<!-- <param value="pn" name="pager.currentPage" type="attach"/>-->
</global-params>
<rules>
<rule controller="city" action="condition2" channel="v2">
@@ -33,5 +33,10 @@
<ladder>/property/avgPrice</ladder>
</ladders>
</rule>
<rule controller="address" action="standard2" channel="v2">
<ladders>
<ladder>/address/standard</ladder>
</ladders>
</rule>
</rules>
</urlrewrite>
@@ -152,7 +152,8 @@ $(document).on('click', '.create-actions03', function () {
$("#buildType").val("8")
$("#houseFloor").show()
$("#floorDesc").hide()
$("#basementAreaLi").hide()
$("#basementAreaLi").show()
$("#basementAreaLi_intr").show()
}
},
{
@@ -163,6 +164,7 @@ $(document).on('click', '.create-actions03', function () {
$("#floorDesc").show()
// $("#houseFloor").hide()
$("#basementAreaLi").show()
$("#basementAreaLi_intr").show()
}
},
{
@@ -173,6 +175,7 @@ $(document).on('click', '.create-actions03', function () {
$("#floorDesc").show()
// $("#houseFloor").hide()
$("#basementAreaLi").show()
$("#basementAreaLi_intr").show()
}
},
{
@@ -183,6 +186,7 @@ $(document).on('click', '.create-actions03', function () {
$("#floorDesc").show()
// $("#houseFloor").hide()
$("#basementAreaLi").show()
$("#basementAreaLi_intr").show()
}
},
{
@@ -193,6 +197,7 @@ $(document).on('click', '.create-actions03', function () {
$("#floorDesc").show()
// $("#houseFloor").hide()
$("#basementAreaLi").show()
$("#basementAreaLi_intr").show()
}
},
{
@@ -203,6 +208,7 @@ $(document).on('click', '.create-actions03', function () {
$("#floorDesc").show()
// $("#houseFloor").hide()
$("#basementAreaLi").show()
$("#basementAreaLi_intr").show()
}
},
{
@@ -213,6 +219,7 @@ $(document).on('click', '.create-actions03', function () {
$("#floorDesc").show()
// $("#houseFloor").hide()
$("#basementAreaLi").show()
$("#basementAreaLi_intr").show()
}
},
{
@@ -223,6 +230,7 @@ $(document).on('click', '.create-actions03', function () {
$("#floorDesc").show()
// $("#houseFloor").hide()
$("#basementAreaLi").hide()
$("#basementAreaLi_intr").hide()
}
},
{
@@ -233,6 +241,7 @@ $(document).on('click', '.create-actions03', function () {
$("#houseFloor").show()
// $("#floorDesc").hide()
$("#basementAreaLi").hide()
$("#basementAreaLi_intr").hide()
}
}
];
@@ -172,6 +172,13 @@
</div>
</div>
</li>
<li id="basementAreaLi_intr" style="color:#999;" class="dn">
<div class="item-content">
<div class="item-inner">
*建筑面积中含有地下附属面积
</div>
</div>
</li>
</ul>
@@ -44,7 +44,7 @@
<#if role?? && role==17 && loanInfoMap??>
<p style="margin-top:-12px;">所属分类:${loanInfoMap.type!}类</p>
<p style="margin-top:-10px;">可贷成数:${loanInfoMap.loan_ratio!}成</p>
<p style="margin-top:-22px;">可贷金额:<strong class="price_final">${(fjtEvaluation.totalPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</strong>万元</p>
<p style="margin-top:-22px;">可贷金额:<strong class="price_final">${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}</strong>万元</p>
<p style="margin-top:-4px;">贷款利率:4.5%</p>
</#if>
<#else>
@@ -88,7 +88,7 @@
<div class="topic inputTopic01" id="inputTopic01a">
<div class="topicTitle topicTitle01">
<span>期望价:</span>
<input class="input_price" type="textArea" name="dealPrice">&nbsp;万元
<input class="input_price" type="textArea" name="dealPrice" <#if userTrades??>disabled</#if>>&nbsp;万元
<a class="submit_btn" id="submitbhPrice" style="border-left: none;">提交</a>
<a class="submit_btn dn" id="refreshbhPrice" style="border-left: none;">刷新</a>
</div>
@@ -11,6 +11,7 @@ import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
@@ -454,22 +455,46 @@ def city = "
// }
//})
ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
//ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
// if(d['A']){
// try{
// def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
// if(dist){
// d['B'] = dist.city
// d['C'] = dist.region
// d['D'] = dist.name
//
// def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
// d['E'] = price
// }
// }catch(Exception e){
// println d['A']
// }
//
// }
//
//})
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d ->
if(d['A']){
try{
def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
if(dist){
d['B'] = dist.city
d['C'] = dist.region
d['D'] = dist.name
def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
d['E'] = price
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def mappingAlias = dist.mapping_alias
if(mappingAlias){
if(mappingAlias.contains(d['B'])){
mappingAlias.remove(d['B'])
DBObject udata = new BasicDBObject();
udata.put("mapping_alias", mappingAlias);
udata.put("m_date", new Date());
DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata))
}else{
println d['A']+"无映射别名"
}
}else{
println d['A']+"映射别名为空"
}
}catch(Exception e){
println d['A']
}else{
println d['A']+"没找到"
}
}
})
@@ -830,6 +830,7 @@ class CommonDatautils {
keys.put("lng", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -20,7 +20,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
// task.regionPriceTrendRun()
task.regionPriceTrendRun()
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202201.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -0,0 +1,163 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import com.fangjia.core.util.DataUtil
import org.apache.commons.lang.math.RandomUtils
class DistrictSelectHouse {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new DistrictSelectHouse();
// task.run()
// task.run2()
task.run3()
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本V1 -跑地址.xlsx",{d->
if(d['A']) {
def dist = CommonDatautils.findDistrict(d['A'],d['B'], d['C'], "district")
if(dist){
d['L'] = dist.address
}
}
},0)
}
def run2(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本.xlsx",{d->
if(d['N']){
def floorInfo = d['J']
def height = d['N']
if(floorInfo && height){
height = height.toInteger()
def mod = (int)(height/3)
def h_start = (height - mod).toInteger()
def random_floor = null
if(floorInfo.contains("低楼层")){
random_floor = random(1, mod)
}else if(floorInfo.contains("中楼层")){
random_floor = random(mod+1, h_start-1)
}else if(floorInfo.contains("高楼层")){
random_floor = random(h_start, height)
}
d['M'] = random_floor
}
}
},0)
}
int random(int i1, int i2) {
int max = Math.max(i1, i2);
int min = Math.min(i1, i2);
return min + RandomUtils.nextInt(max - min + 1);
}
def run(){
def city = "重庆"
def weekend = "2022-03-06"
def thirdPriceMap = findNearThirdPartyPrice(city)
def distStats = findCreditHightDistrict2(city, weekend, thirdPriceMap)
findAllHouse(distStats);
}
def findAllHouse(distStats){
def houses = []
distStats.each{k,v->
def arrs = k.split("#")
def list = findHouses(arrs[0],arrs[1], arrs[2], 20220201,20220401)
if(list){
houses.addAll(list)
}
}
if(houses){
def rets = []
for(def h : houses){
def row = []
row << h.city
row << h.region
row << h.district_name
row << h.area
row << h.avg_price
row << h.total_price
row << h.direction
row << h.estate_type
row << h.fitment
row << h.floor_info
row << h.room
row << h.hall
row << h.floor
row << h.height
row << h.p_date
rets << row
}
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆样本.xlsx", rets, "数据")
}
}
def findHouses(def city, def region, def name, def startDate, def endDate){
String sql = "select * from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and region='${region}' and district_name='${name}' and p_date >= ${startDate} and p_date<${endDate}"
def houses = []
ElasticUtils.executeSqlQuery(sql,{ data->
houses << data
})
if(houses.size()>10){
houses = houses[0..9]
}
return houses
}
def findNearThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', 202108))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
def findCreditHightDistrict2(def city, def weekend, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice) {
def precent = (thirdPrice - d.avg_price) * 100 / d.avg_price
if (Math.abs(precent) >= 0 && Math.abs(precent) <= 10) {
retMaps.put(d.city +"#"+d.region+"#"+d.district_name, d.avg_price)
}
}
}
return retMaps
}
}
@@ -1,15 +1,18 @@
package com.fangjia.script.gj.level
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil
@@ -18,7 +21,11 @@ import groovy.sql.Sql
import net.sf.json.JSONObject
import org.apache.commons.httpclient.util.DateUtil
import org.apache.commons.lang.StringUtils
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
import org.apache.commons.math3.stat.descriptive.moment.Mean
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -44,9 +51,9 @@ class DistrictIndexBatch extends BaseBatch{
dataSource.setProperties(connectionProperties)
}
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc"
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -78,7 +85,9 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price)
calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
println city +" "+city_avg_total_price
}
@@ -110,46 +119,25 @@ class DistrictIndexBatch extends BaseBatch{
def total_price = getHousePrice(d, dists, priceMap, citycode);
def avg_cost = getAvgCost(d, costDatas)
def total_price_index = 0
if (total_price) {
total_price_index = 286.0134196 * (total_price - city_avg_total_price) / 313.3693508
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
}
def avg_charge_index = 0
if (avg_charge) {
avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
}
def rank_index = 0
if (rank) {
rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
}
def avg_cost_index = 0
if (avg_cost) {
avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
}
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
def map = [:]
map.id = d._id.toString()
map.city = d.city
map.region = d.region
map.name = d.name
map.avg_total_price = total_price_index
map.avg_estate_charge = avg_charge
map.price_rank = rank
map.avg_cost = avg_cost
map.comp_value = potentiality
map.month = "2022-01"
map.total_price = total_price
//map.comp_value = potentiality
//map.avg_total_price = total_price_index
map.month = "2022-03"
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
def newUrl = targetUrl +"/"+map.id
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price
} catch (Exception e) {
@@ -163,16 +151,266 @@ class DistrictIndexBatch extends BaseBatch{
latch.await();
def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
def fileName = "/home/gj/potentiality_${city}.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
// def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
// def fileName = "/home/gj/potentiality_${city}.dat"
// if(CommonDatautils.isWindow()){
// fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
// }
// FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
// // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
}
def calcuteIndex(city, dists, citycode, city_avg_total_price){
def detailMap = findAllDistrictDatas(city)
def avgMap = calcuteAvgValues(detailMap);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def avg_total_price = avgMap.avg_price
def avg_charge = avgMap.avg_charge
def avg_cost = avgMap.avg_cost
def avg_rank = avgMap.avg_rank
def sd_avg_price = avgMap.sd_avg_price
def sd_avg_charge = avgMap.sd_avg_charge
def sd_avg_cost = avgMap.sd_avg_cost
def sd_avg_rank = avgMap.sd_avg_rank
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def rMap = detailMap.get(d._id.toString())
if(rMap){
def total_price_index = 0
if (rMap.total_price) {
// total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / 313.3693508
total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / sd_avg_price
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
}
def avg_charge_index = 0
if (rMap.avg_estate_charge) {
// avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / 0.772635241
avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / sd_avg_charge
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
}
def rank_index = 0
if (rMap.price_rank) {
// rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / 0.261907636
rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / sd_avg_rank
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
}
def avg_cost_index = 0
if (rMap.avg_cost) {
// avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / 31.3526959
avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / sd_avg_cost
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
}
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
def map = [:]
map.id = d._id.toString()
map.city = d.city
map.region = d.region
map.name = d.name
map.avg_estate_charge = rMap.avg_estate_charge
map.price_rank = rMap.price_rank
map.avg_cost = rMap.avg_cost
map.total_price = rMap.total_price
map.comp_value = potentiality
map.avg_total_price = total_price_index
map.month = "2022-03"
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
//修改多了会报错
// def newUrl = targetUrl +"_update_by_query"
// def must = []
// def term1 = ["match_phrase":["id":d._id.toString()]]
// must << term1
// def source = "ctx._source.comp_value="+potentiality+";ctx._source.avg_total_price="+total_price_index+";ctx._source.month='2022-03'"+";ctx._source.m_date='"+DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")+"'"
// def params = ["script":["source":source,"lang": "painless"], "query":["bool":["must":must]]]
// httpSaveData(newUrl, params);
println totalCount.get() +" "+d.city+" "+d.region+" "+d.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown();
}
}
})
}
latch.await();
}
def findAllDistrictDatas(def city){
def detailMap = [:]
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
for(def rs : houses){
def id = rs.id
def rMap = detailMap.get(id)
if(!rMap){
rMap = [:]
detailMap.put(id, rMap)
}
rMap.put("total_price", rs.total_price)
rMap.put("price_rank", rs.price_rank)
rMap.put("avg_cost", rs.avg_cost)
rMap.put("avg_estate_charge", rs.avg_estate_charge)
}
return detailMap;
}
def findDistrictDatasByCity(def city){
def must = []
def term1 = ["term":["city":city]]
must << term1
// def params = ["query":["bool":["must":must]]]
def params = ["query":["match":["city":city]]]
def s = httpSaveData(targetUrl+"/_search", params);
def total = 0;
if(s){
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
total = hits.total
}
}
}
def calcuteAvgValues(detailMap){
def prices = []
def ranks = []
def costs = []
def charges = []
detailMap.each{k,v->
if(v.total_price){
prices << v.total_price
}
if(v.price_rank){
ranks << v.price_rank
}
if(v.avg_cost){
costs << v.avg_cost
}
if(v.avg_estate_charge){
charges << v.avg_estate_charge
}
}
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
def avg_price = null
def dx_avg_price = null
def sd_avg_price = null
if(prices){
def v = prices.sum()/prices.size()
avg_price = DataHandleUtils.getDoubleRound(6, v)
double pow_sum = 0
prices.each {p->
def sod = Math.abs(p - avg_price)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_price = DataHandleUtils.getDoubleRound(6,pow_sum/prices.size())
sd_avg_price = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_price))
}
def avg_rank = null
def dx_avg_rank = null
def sd_avg_rank = null
if(ranks){
def v = ranks.sum()/ranks.size()
avg_rank = DataHandleUtils.getDoubleRound(6, v)
def pow_sum = 0;
ranks.each {p->
def sod = Math.abs(p - avg_rank)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_rank = DataHandleUtils.getDoubleRound(6,pow_sum/ranks.size())
sd_avg_rank = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_rank))
}
def avg_cost = null
def dx_avg_cost = null
def sd_avg_cost = null
if(costs){
def v = costs.sum()/costs.size()
avg_cost = DataHandleUtils.getDoubleRound(6, v)
double pow_sum = 0;
costs.each {p->
def sod = Math.abs(p - avg_cost)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_cost = DataHandleUtils.getDoubleRound(6,pow_sum/costs.size())
sd_avg_cost = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_cost))
}
def avg_charge = null
def dx_avg_charge = null
def sd_avg_charge = null
if(charges){
def v = charges.sum()/charges.size()
avg_charge = DataHandleUtils.getDoubleRound(6, v)
def pow_sum = 0;
charges.each {p->
def sod = Math.abs(p - avg_charge)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_charge = DataHandleUtils.getDoubleRound(6,pow_sum/charges.size())
sd_avg_charge = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_charge))
}
def avgMap = [:]
avgMap.put("avg_rank", avg_rank)
avgMap.put("avg_price", avg_price)
avgMap.put("avg_cost", avg_cost)
avgMap.put("avg_charge", avg_charge)
avgMap.put("dx_avg_rank", dx_avg_rank)
avgMap.put("dx_avg_price", dx_avg_price)
avgMap.put("dx_avg_cost", dx_avg_cost)
avgMap.put("dx_avg_charge", dx_avg_charge)
avgMap.put("sd_avg_rank", sd_avg_rank)
avgMap.put("sd_avg_price", sd_avg_price)
avgMap.put("sd_avg_cost", sd_avg_cost)
avgMap.put("sd_avg_charge", sd_avg_charge)
return avgMap;
}
//方差s^2=[(x1-x)^2 +...(xn-x)^2]/n 或者s^2=[(x1-x)^2 +...(xn-x)^2]/(n-1)
public double Variance(def x) {
int m=x.size();
double sum=0;
for(int i=0;i<m;i++){//求和
sum+=x[i];
}
double dAve=sum/m;//求平均值
double dVar=0;
for(int i=0;i<m;i++){//求方差
dVar+=(x[i]-dAve)*(x[i]-dAve);
}
return dVar/m;
}
def getAvgTotalPrice(city, startDate, endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT avg(total_price) as avg_total_price FROM \"seawater-20*\" WHERE 1=1")
@@ -496,6 +734,7 @@ class DistrictIndexBatch extends BaseBatch{
def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
@@ -508,7 +747,7 @@ class DistrictIndexBatch extends BaseBatch{
String param = null;
BufferedWriter mWriter = null;
InputStreamReader inStream = null;
StringBuilder buffer = new StringBuilder();
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
@@ -547,7 +786,7 @@ class DistrictIndexBatch extends BaseBatch{
} catch (Exception e) {
e.printStackTrace()
}
return null;
return buffer.toString();
}
}
File diff suppressed because one or more lines are too long
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
//"fang.com":"搜房",
def sourceMap = ["cityhouse.cn":"城市房产"]
def sourceMap = ["jiwu.com":"吉屋","ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()