小区潜力指数

This commit is contained in:
2022-03-11 15:32:03 +08:00
parent 9cef9a0a2b
commit 301bcb0984
5 changed files with 441 additions and 330 deletions
@@ -830,6 +830,7 @@ class CommonDatautils {
keys.put("lng", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -20,7 +20,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
// task.regionPriceTrendRun()
task.regionPriceTrendRun()
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
@@ -1,15 +1,18 @@
package com.fangjia.script.gj.level
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil
@@ -18,7 +21,11 @@ import groovy.sql.Sql
import net.sf.json.JSONObject
import org.apache.commons.httpclient.util.DateUtil
import org.apache.commons.lang.StringUtils
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
import org.apache.commons.math3.stat.descriptive.moment.Mean
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -44,9 +51,9 @@ class DistrictIndexBatch extends BaseBatch{
dataSource.setProperties(connectionProperties)
}
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc"
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -78,7 +85,9 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price)
calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
println city +" "+city_avg_total_price
}
@@ -110,46 +119,25 @@ class DistrictIndexBatch extends BaseBatch{
def total_price = getHousePrice(d, dists, priceMap, citycode);
def avg_cost = getAvgCost(d, costDatas)
def total_price_index = 0
if (total_price) {
total_price_index = 286.0134196 * (total_price - city_avg_total_price) / 313.3693508
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
}
def avg_charge_index = 0
if (avg_charge) {
avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
}
def rank_index = 0
if (rank) {
rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
}
def avg_cost_index = 0
if (avg_cost) {
avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
}
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
def map = [:]
map.id = d._id.toString()
map.city = d.city
map.region = d.region
map.name = d.name
map.avg_total_price = total_price_index
map.avg_estate_charge = avg_charge
map.price_rank = rank
map.avg_cost = avg_cost
map.comp_value = potentiality
map.month = "2022-01"
map.total_price = total_price
//map.comp_value = potentiality
//map.avg_total_price = total_price_index
map.month = "2022-03"
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
def newUrl = targetUrl +"/"+map.id
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price
} catch (Exception e) {
@@ -163,16 +151,266 @@ class DistrictIndexBatch extends BaseBatch{
latch.await();
def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
def fileName = "/home/gj/potentiality_${city}.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
// def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
// def fileName = "/home/gj/potentiality_${city}.dat"
// if(CommonDatautils.isWindow()){
// fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
// }
// FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
// // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
}
def calcuteIndex(city, dists, citycode, city_avg_total_price){
def detailMap = findAllDistrictDatas(city)
def avgMap = calcuteAvgValues(detailMap);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def avg_total_price = avgMap.avg_price
def avg_charge = avgMap.avg_charge
def avg_cost = avgMap.avg_cost
def avg_rank = avgMap.avg_rank
def sd_avg_price = avgMap.sd_avg_price
def sd_avg_charge = avgMap.sd_avg_charge
def sd_avg_cost = avgMap.sd_avg_cost
def sd_avg_rank = avgMap.sd_avg_rank
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def rMap = detailMap.get(d._id.toString())
if(rMap){
def total_price_index = 0
if (rMap.total_price) {
// total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / 313.3693508
total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / sd_avg_price
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
}
def avg_charge_index = 0
if (rMap.avg_estate_charge) {
// avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / 0.772635241
avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / sd_avg_charge
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
}
def rank_index = 0
if (rMap.price_rank) {
// rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / 0.261907636
rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / sd_avg_rank
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
}
def avg_cost_index = 0
if (rMap.avg_cost) {
// avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / 31.3526959
avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / sd_avg_cost
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
}
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
def map = [:]
map.id = d._id.toString()
map.city = d.city
map.region = d.region
map.name = d.name
map.avg_estate_charge = rMap.avg_estate_charge
map.price_rank = rMap.price_rank
map.avg_cost = rMap.avg_cost
map.total_price = rMap.total_price
map.comp_value = potentiality
map.avg_total_price = total_price_index
map.month = "2022-03"
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
//修改多了会报错
// def newUrl = targetUrl +"_update_by_query"
// def must = []
// def term1 = ["match_phrase":["id":d._id.toString()]]
// must << term1
// def source = "ctx._source.comp_value="+potentiality+";ctx._source.avg_total_price="+total_price_index+";ctx._source.month='2022-03'"+";ctx._source.m_date='"+DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")+"'"
// def params = ["script":["source":source,"lang": "painless"], "query":["bool":["must":must]]]
// httpSaveData(newUrl, params);
println totalCount.get() +" "+d.city+" "+d.region+" "+d.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown();
}
}
})
}
latch.await();
}
def findAllDistrictDatas(def city){
def detailMap = [:]
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
for(def rs : houses){
def id = rs.id
def rMap = detailMap.get(id)
if(!rMap){
rMap = [:]
detailMap.put(id, rMap)
}
rMap.put("total_price", rs.total_price)
rMap.put("price_rank", rs.price_rank)
rMap.put("avg_cost", rs.avg_cost)
rMap.put("avg_estate_charge", rs.avg_estate_charge)
}
return detailMap;
}
def findDistrictDatasByCity(def city){
def must = []
def term1 = ["term":["city":city]]
must << term1
// def params = ["query":["bool":["must":must]]]
def params = ["query":["match":["city":city]]]
def s = httpSaveData(targetUrl+"/_search", params);
def total = 0;
if(s){
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
total = hits.total
}
}
}
def calcuteAvgValues(detailMap){
def prices = []
def ranks = []
def costs = []
def charges = []
detailMap.each{k,v->
if(v.total_price){
prices << v.total_price
}
if(v.price_rank){
ranks << v.price_rank
}
if(v.avg_cost){
costs << v.avg_cost
}
if(v.avg_estate_charge){
charges << v.avg_estate_charge
}
}
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
def avg_price = null
def dx_avg_price = null
def sd_avg_price = null
if(prices){
def v = prices.sum()/prices.size()
avg_price = DataHandleUtils.getDoubleRound(6, v)
double pow_sum = 0
prices.each {p->
def sod = Math.abs(p - avg_price)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_price = DataHandleUtils.getDoubleRound(6,pow_sum/prices.size())
sd_avg_price = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_price))
}
def avg_rank = null
def dx_avg_rank = null
def sd_avg_rank = null
if(ranks){
def v = ranks.sum()/ranks.size()
avg_rank = DataHandleUtils.getDoubleRound(6, v)
def pow_sum = 0;
ranks.each {p->
def sod = Math.abs(p - avg_rank)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_rank = DataHandleUtils.getDoubleRound(6,pow_sum/ranks.size())
sd_avg_rank = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_rank))
}
def avg_cost = null
def dx_avg_cost = null
def sd_avg_cost = null
if(costs){
def v = costs.sum()/costs.size()
avg_cost = DataHandleUtils.getDoubleRound(6, v)
double pow_sum = 0;
costs.each {p->
def sod = Math.abs(p - avg_cost)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_cost = DataHandleUtils.getDoubleRound(6,pow_sum/costs.size())
sd_avg_cost = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_cost))
}
def avg_charge = null
def dx_avg_charge = null
def sd_avg_charge = null
if(charges){
def v = charges.sum()/charges.size()
avg_charge = DataHandleUtils.getDoubleRound(6, v)
def pow_sum = 0;
charges.each {p->
def sod = Math.abs(p - avg_charge)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_charge = DataHandleUtils.getDoubleRound(6,pow_sum/charges.size())
sd_avg_charge = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_charge))
}
def avgMap = [:]
avgMap.put("avg_rank", avg_rank)
avgMap.put("avg_price", avg_price)
avgMap.put("avg_cost", avg_cost)
avgMap.put("avg_charge", avg_charge)
avgMap.put("dx_avg_rank", dx_avg_rank)
avgMap.put("dx_avg_price", dx_avg_price)
avgMap.put("dx_avg_cost", dx_avg_cost)
avgMap.put("dx_avg_charge", dx_avg_charge)
avgMap.put("sd_avg_rank", sd_avg_rank)
avgMap.put("sd_avg_price", sd_avg_price)
avgMap.put("sd_avg_cost", sd_avg_cost)
avgMap.put("sd_avg_charge", sd_avg_charge)
return avgMap;
}
//方差s^2=[(x1-x)^2 +...(xn-x)^2]/n 或者s^2=[(x1-x)^2 +...(xn-x)^2]/(n-1)
public double Variance(def x) {
int m=x.size();
double sum=0;
for(int i=0;i<m;i++){//求和
sum+=x[i];
}
double dAve=sum/m;//求平均值
double dVar=0;
for(int i=0;i<m;i++){//求方差
dVar+=(x[i]-dAve)*(x[i]-dAve);
}
return dVar/m;
}
def getAvgTotalPrice(city, startDate, endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT avg(total_price) as avg_total_price FROM \"seawater-20*\" WHERE 1=1")
@@ -496,6 +734,7 @@ class DistrictIndexBatch extends BaseBatch{
def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
@@ -508,7 +747,7 @@ class DistrictIndexBatch extends BaseBatch{
String param = null;
BufferedWriter mWriter = null;
InputStreamReader inStream = null;
StringBuilder buffer = new StringBuilder();
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
@@ -547,7 +786,7 @@ class DistrictIndexBatch extends BaseBatch{
} catch (Exception e) {
e.printStackTrace()
}
return null;
return buffer.toString();
}
}
File diff suppressed because one or more lines are too long
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
//"fang.com":"搜房",
def sourceMap = ["cityhouse.cn":"城市房产"]
def sourceMap = ["jiwu.com":"吉屋","ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()