小区潜力指数

This commit is contained in:
2022-03-11 15:32:03 +08:00
parent 9cef9a0a2b
commit 301bcb0984
5 changed files with 441 additions and 330 deletions
@@ -830,6 +830,7 @@ class CommonDatautils {
keys.put("lng", 1); keys.put("lng", 1);
keys.put("name_cat", 1); keys.put("name_cat", 1);
keys.put("finish_building_date", 1) keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys); DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){ if(cursor){
for(def obj : cursor){ for(def obj : cursor){
@@ -20,7 +20,7 @@ class FzDataResultStat {
static main(args) { static main(args) {
def task = new FzDataResultStat() def task = new FzDataResultStat()
task.priceMarkupRun(); task.priceMarkupRun();
// task.regionPriceTrendRun() task.regionPriceTrendRun()
} }
def priceMarkupRun(){ def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
@@ -1,15 +1,18 @@
package com.fangjia.script.gj.level package com.fangjia.script.gj.level
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil import com.fangjia.script.util.JacksonUtil
@@ -18,7 +21,11 @@ import groovy.sql.Sql
import net.sf.json.JSONObject import net.sf.json.JSONObject
import org.apache.commons.httpclient.util.DateUtil import org.apache.commons.httpclient.util.DateUtil
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
import org.apache.commons.math3.stat.descriptive.moment.Mean
import org.bson.types.ObjectId import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
@@ -44,9 +51,9 @@ class DistrictIndexBatch extends BaseBatch{
dataSource.setProperties(connectionProperties) dataSource.setProperties(connectionProperties)
} }
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc" static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -78,7 +85,9 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913 city_avg_total_price = 531.2615913
} }
if(city_avg_total_price){ if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price) calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
} }
println city +" "+city_avg_total_price println city +" "+city_avg_total_price
} }
@@ -110,46 +119,25 @@ class DistrictIndexBatch extends BaseBatch{
def total_price = getHousePrice(d, dists, priceMap, citycode); def total_price = getHousePrice(d, dists, priceMap, citycode);
def avg_cost = getAvgCost(d, costDatas) def avg_cost = getAvgCost(d, costDatas)
def total_price_index = 0
if (total_price) {
total_price_index = 286.0134196 * (total_price - city_avg_total_price) / 313.3693508
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
}
def avg_charge_index = 0
if (avg_charge) {
avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
}
def rank_index = 0
if (rank) {
rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
}
def avg_cost_index = 0
if (avg_cost) {
avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
}
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
def map = [:] def map = [:]
map.id = d._id.toString() map.id = d._id.toString()
map.city = d.city map.city = d.city
map.region = d.region map.region = d.region
map.name = d.name map.name = d.name
map.avg_total_price = total_price_index
map.avg_estate_charge = avg_charge map.avg_estate_charge = avg_charge
map.price_rank = rank map.price_rank = rank
map.avg_cost = avg_cost map.avg_cost = avg_cost
map.comp_value = potentiality map.total_price = total_price
map.month = "2022-01" //map.comp_value = potentiality
//map.avg_total_price = total_price_index
map.month = "2022-03"
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] //datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
def newUrl = targetUrl +"/"+map.id def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map) httpSaveData(newUrl, map)
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price
} catch (Exception e) { } catch (Exception e) {
@@ -163,16 +151,266 @@ class DistrictIndexBatch extends BaseBatch{
latch.await(); latch.await();
def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] // def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
def fileName = "/home/gj/potentiality_${city}.dat" // def fileName = "/home/gj/potentiality_${city}.dat"
if(CommonDatautils.isWindow()){ // if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\评级\\${city}综合潜力.dat" // fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
// }
// FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
// // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
}
def calcuteIndex(city, dists, citycode, city_avg_total_price){
def detailMap = findAllDistrictDatas(city)
def avgMap = calcuteAvgValues(detailMap);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def avg_total_price = avgMap.avg_price
def avg_charge = avgMap.avg_charge
def avg_cost = avgMap.avg_cost
def avg_rank = avgMap.avg_rank
def sd_avg_price = avgMap.sd_avg_price
def sd_avg_charge = avgMap.sd_avg_charge
def sd_avg_cost = avgMap.sd_avg_cost
def sd_avg_rank = avgMap.sd_avg_rank
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def rMap = detailMap.get(d._id.toString())
if(rMap){
def total_price_index = 0
if (rMap.total_price) {
// total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / 313.3693508
total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / sd_avg_price
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
}
def avg_charge_index = 0
if (rMap.avg_estate_charge) {
// avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / 0.772635241
avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / sd_avg_charge
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
}
def rank_index = 0
if (rMap.price_rank) {
// rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / 0.261907636
rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / sd_avg_rank
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
}
def avg_cost_index = 0
if (rMap.avg_cost) {
// avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / 31.3526959
avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / sd_avg_cost
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
}
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
def map = [:]
map.id = d._id.toString()
map.city = d.city
map.region = d.region
map.name = d.name
map.avg_estate_charge = rMap.avg_estate_charge
map.price_rank = rMap.price_rank
map.avg_cost = rMap.avg_cost
map.total_price = rMap.total_price
map.comp_value = potentiality
map.avg_total_price = total_price_index
map.month = "2022-03"
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
//修改多了会报错
// def newUrl = targetUrl +"_update_by_query"
// def must = []
// def term1 = ["match_phrase":["id":d._id.toString()]]
// must << term1
// def source = "ctx._source.comp_value="+potentiality+";ctx._source.avg_total_price="+total_price_index+";ctx._source.month='2022-03'"+";ctx._source.m_date='"+DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")+"'"
// def params = ["script":["source":source,"lang": "painless"], "query":["bool":["must":must]]]
// httpSaveData(newUrl, params);
println totalCount.get() +" "+d.city+" "+d.region+" "+d.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown();
}
}
})
}
latch.await();
}
def findAllDistrictDatas(def city){
def detailMap = [:]
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
for(def rs : houses){
def id = rs.id
def rMap = detailMap.get(id)
if(!rMap){
rMap = [:]
detailMap.put(id, rMap)
}
rMap.put("total_price", rs.total_price)
rMap.put("price_rank", rs.price_rank)
rMap.put("avg_cost", rs.avg_cost)
rMap.put("avg_estate_charge", rs.avg_estate_charge)
}
return detailMap;
}
def findDistrictDatasByCity(def city){
def must = []
def term1 = ["term":["city":city]]
must << term1
// def params = ["query":["bool":["must":must]]]
def params = ["query":["match":["city":city]]]
def s = httpSaveData(targetUrl+"/_search", params);
def total = 0;
if(s){
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
total = hits.total
}
}
}
def calcuteAvgValues(detailMap){
def prices = []
def ranks = []
def costs = []
def charges = []
detailMap.each{k,v->
if(v.total_price){
prices << v.total_price
}
if(v.price_rank){
ranks << v.price_rank
}
if(v.avg_cost){
costs << v.avg_cost
}
if(v.avg_estate_charge){
charges << v.avg_estate_charge
}
} }
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false); def avg_price = null
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") def dx_avg_price = null
def sd_avg_price = null
if(prices){
def v = prices.sum()/prices.size()
avg_price = DataHandleUtils.getDoubleRound(6, v)
double pow_sum = 0
prices.each {p->
def sod = Math.abs(p - avg_price)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_price = DataHandleUtils.getDoubleRound(6,pow_sum/prices.size())
sd_avg_price = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_price))
}
def avg_rank = null
def dx_avg_rank = null
def sd_avg_rank = null
if(ranks){
def v = ranks.sum()/ranks.size()
avg_rank = DataHandleUtils.getDoubleRound(6, v)
def pow_sum = 0;
ranks.each {p->
def sod = Math.abs(p - avg_rank)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_rank = DataHandleUtils.getDoubleRound(6,pow_sum/ranks.size())
sd_avg_rank = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_rank))
}
def avg_cost = null
def dx_avg_cost = null
def sd_avg_cost = null
if(costs){
def v = costs.sum()/costs.size()
avg_cost = DataHandleUtils.getDoubleRound(6, v)
double pow_sum = 0;
costs.each {p->
def sod = Math.abs(p - avg_cost)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_cost = DataHandleUtils.getDoubleRound(6,pow_sum/costs.size())
sd_avg_cost = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_cost))
}
def avg_charge = null
def dx_avg_charge = null
def sd_avg_charge = null
if(charges){
def v = charges.sum()/charges.size()
avg_charge = DataHandleUtils.getDoubleRound(6, v)
def pow_sum = 0;
charges.each {p->
def sod = Math.abs(p - avg_charge)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_charge = DataHandleUtils.getDoubleRound(6,pow_sum/charges.size())
sd_avg_charge = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_charge))
}
def avgMap = [:]
avgMap.put("avg_rank", avg_rank)
avgMap.put("avg_price", avg_price)
avgMap.put("avg_cost", avg_cost)
avgMap.put("avg_charge", avg_charge)
avgMap.put("dx_avg_rank", dx_avg_rank)
avgMap.put("dx_avg_price", dx_avg_price)
avgMap.put("dx_avg_cost", dx_avg_cost)
avgMap.put("dx_avg_charge", dx_avg_charge)
avgMap.put("sd_avg_rank", sd_avg_rank)
avgMap.put("sd_avg_price", sd_avg_price)
avgMap.put("sd_avg_cost", sd_avg_cost)
avgMap.put("sd_avg_charge", sd_avg_charge)
return avgMap;
} }
//方差s^2=[(x1-x)^2 +...(xn-x)^2]/n 或者s^2=[(x1-x)^2 +...(xn-x)^2]/(n-1)
public double Variance(def x) {
int m=x.size();
double sum=0;
for(int i=0;i<m;i++){//求和
sum+=x[i];
}
double dAve=sum/m;//求平均值
double dVar=0;
for(int i=0;i<m;i++){//求方差
dVar+=(x[i]-dAve)*(x[i]-dAve);
}
return dVar/m;
}
def getAvgTotalPrice(city, startDate, endDate){ def getAvgTotalPrice(city, startDate, endDate){
final StringBuffer sqlBuffer = new StringBuffer() final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT avg(total_price) as avg_total_price FROM \"seawater-20*\" WHERE 1=1") sqlBuffer.append("SELECT avg(total_price) as avg_total_price FROM \"seawater-20*\" WHERE 1=1")
@@ -496,6 +734,7 @@ class DistrictIndexBatch extends BaseBatch{
def doPost(def targetUrl, def headers, def params, def isJson, def charset){ def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{ try{
URL url = new URL(targetUrl); URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); HttpURLConnection conn = (HttpURLConnection) url.openConnection();
@@ -508,7 +747,7 @@ class DistrictIndexBatch extends BaseBatch{
String param = null; String param = null;
BufferedWriter mWriter = null; BufferedWriter mWriter = null;
InputStreamReader inStream = null; InputStreamReader inStream = null;
StringBuilder buffer = new StringBuilder();
if (headers != null && headers.keySet().size() > 0) { if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) { for (String head : headers.keySet()) {
@@ -547,7 +786,7 @@ class DistrictIndexBatch extends BaseBatch{
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace() e.printStackTrace()
} }
return null; return buffer.toString();
} }
} }
File diff suppressed because one or more lines are too long
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
//"fang.com":"搜房", //"fang.com":"搜房",
def sourceMap = ["cityhouse.cn":"城市房产"] def sourceMap = ["jiwu.com":"吉屋","ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
static main(args) { static main(args) {
def task = new SynchDistrictPrice() def task = new SynchDistrictPrice()
task.run() task.run()