Merge branch 'script_gj' into 'master'

7个城市

See merge request !738
This commit is contained in:
2023-10-23 20:47:56 +08:00
10 changed files with 266 additions and 24 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/** /**
* 深圳公众号成交数据录入 * 深圳公众号成交数据录入
*/ */
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳8月份二手房成交数据汇总.xls", 0, 1) def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -58,10 +58,16 @@ for(def d : datas){
}catch(Exception e){ }catch(Exception e){
println "room error" println "room error"
} }
map.hall = arrs[1].toString().toInteger() if(arrs[1].toString()){
map.toilet = arrs[3].toString().toInteger() map.hall = arrs[1].toString().toInteger()
}
if(arrs.size()==4 && arrs[3].toString()){
map.toilet = arrs[3].toString().toInteger()
}
}else{ }else{
map.room = roomStr.toString().toDouble().toInteger() if(roomStr.toString()){
map.room = roomStr.toString().toDouble().toInteger()
}
} }
} }
@@ -74,7 +80,7 @@ for(def d : datas){
if(trade_date){ if(trade_date){
map.trade_date = trade_date map.trade_date = trade_date
}else{ }else{
def trade_date1 = DateUtils.getRandomDate("2023-08-01", "2023-08-31") def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
} }
@@ -311,7 +311,7 @@ class BigdataCommunityBatch extends BaseBatch{
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def index = 0 def index = 0
sqlInstance.withBatch(10000, sql){ ps-> sqlInstance.withBatch(50000, sql){ ps->
for(def r : newRows) { for(def r : newRows) {
index++ index++
try{ try{
@@ -63,6 +63,7 @@ class BigdataDealBatch extends BaseBatch{
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys; // def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
citys = ['广州','重庆','南京','石家庄','廊坊','佛山','无锡','哈尔滨']
if(args && args.size()>=2){ if(args && args.size()>=2){
curDate = args[1].toString().toInteger() curDate = args[1].toString().toInteger()
} }
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-08-25", "2023-09-27") def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-24")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) { static main(args) {
def task = new TradeStatLog() def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量 task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交 // task.exportTrade()//导出未入库成交
// task.thirdPartyStat() // task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR) // println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat() // task.averageStat()
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){ def run(){
def citys = CcbCommonUtils.citys_all; def citys = CcbCommonUtils.citys_all;
def rets = [] def rets = []
rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] rets << ["城市","23年7月总量","23年8月总量","23年9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01" //def startDate = "2021-05-01"
def startDate = "2023-05-01" def startDate = "2023-05-01"
def date = "2023-06-01" def date = "2023-08-01"
def endDate = "2023-07-01" def endDate = "2023-09-01"
def newDate = "2023-08-01" def newDate = "2023-10-01"
for(def city : citys){ for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -1740,14 +1740,14 @@ class CommonDatautils {
} }
public static def queryDistrictPrice(city, region, name, weekend){ public static def queryFirstDistrictPrice(city, region, name){
DBObject queryCondition = new BasicDBObject() DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city); queryCondition.put("city", city);
queryCondition.put("region", region); queryCondition.put("region", region);
queryCondition.put("district_name", name); queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend); // queryCondition.put("weekend", weekend);
DBCursor cur = DIST_STAT_COLL.find(queryCondition) DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", -1)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def dbo = null; def dbo = null;
while(cur.hasNext()) { while(cur.hasNext()) {
@@ -2215,6 +2215,21 @@ class CommonDatautils {
} }
public static def queryAllDistrictPrice(city, startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put('weekend', startDate);
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rMap = [:];
while(cur.hasNext()) {
DBObject dbo = cur.next()
rMap.put(city + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
}
return rMap;
}
public static def queryAllDistrictPrice(startDate){ public static def queryAllDistrictPrice(startDate){
DBObject queryCondition = new BasicDBObject() DBObject queryCondition = new BasicDBObject()
queryCondition.put('weekend', startDate); queryCondition.put('weekend', startDate);
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-08-01" static def statDate = "2023-09-01"
static main(args) { static main(args) {
def task = new YouDaDataParser(); def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo // task.newHouseparse();//友达一手房存入mongo
task.esfParser("D:\\数据\\友达\\2023年9月\\查询1.DBF");//友达二手房存入mongo // task.esfParser("D:\\数据\\友达\\2023年10月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es // task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -9,8 +9,19 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import groovy.sql.Sql import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 价格对比,第三方,成交,建行价格,自已家价格
*/
class AddPriceTest { class AddPriceTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver") "postgres", "postgres", "org.postgresql.Driver")
@@ -26,10 +37,120 @@ class AddPriceTest {
// addGXDPrice(filePath) // addGXDPrice(filePath)
// addCcbPrice(filePath) // addCcbPrice(filePath)
// addThirdPrice(filePath) // addThirdPrice(filePath)
addFjwPrice(filePath) // addFjwPrice(filePath)
exportPkPrice()
// addRegion()
} }
def addRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
d['C'] = dist.region
})
}
def exportPkPrice(){
//"南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安",
def citys = ["宿迁"]
for(def c : citys){
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","alias"])
println c+" district:"+districts.size()
def fjwPriceMap =[:]// CommonDatautils.queryAllDistrictPrice(c, DateUtils.parseFromTightByPattern("2023-10-08","yyyy-MM-dd"))
def datas = addAllPrice(districts, fjwPriceMap)
ExcelUtils.write("D:\\数据\\上海农总行\\江苏省价格比对.xlsx", datas, c)
println c +" 完成"
}
}
def addAllPrice(districts, fjwPriceMap){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def rets = new CopyOnWriteArrayList()
rets << ["id", "城市","区域","名称","建行价格","计算方式","可信度","房价网ToB价格","计算方式","可信度","近6个月成交均价"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createPriceData(cityCodeDataMap, dist, fjwPriceMap)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createPriceData(cityCodeDataMap, d, fjwPriceMap){
def row = []
row << d._id.toString()
row << d.city
row << d.region
row << d.name
def cityId = cityCodeDataMap.get(d.city)
def map = findAvgPrice(d._id.toString(), cityId)
if(map){
row << map.avg_price
row << map.calc_type
row << map.credit_value
}else{
row << ""
row << ""
row << ""
}
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
// def dbo = fjwPriceMap.get(d.city+"##"+d.region+"##"+d.name)
if(dbo){
row << dbo.avg_price
if(dbo.value_type == 1){
row << "指导价"
}else if(dbo.value_type == 2){
row << "自身样本"
}else if(dbo.value_type == 3){
row << "第三方"
}else if(dbo.value_type == 4){
row << "交易数据"
}else if(dbo.value_type == 5){
row << "周边楼盘"
}else if(dbo.value_type == 6){
row << "推导"
}else if(dbo.value_type == 9){
row << "神经网络"
}else if(dbo.value_type == 10){
row << "无方式"
}else{
row << ""
}
row << dbo.credit
}else{
row << ""
row << ""
row << ""
}
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-09-01')
if(tradePrice){
row << tradePrice.toInteger()
}else{
row << ''
}
return row;
}
def addCcbPrice(filepath){ def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -2,12 +2,14 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data import com.fangjia.script.xingyebank.Data
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CopyOnWriteArrayList
@@ -30,14 +32,16 @@ class PoiWash {
public static def townsMap = [:] public static def townsMap = [:]
public static def regions = [] public static def regions = []
public static def nameKeys = null
static main(args) { static main(args) {
def task = new PoiWash() def task = new PoiWash()
// task.updateId() // task.updateId()
// task.run() // task.run()
// task.synch() // task.synch()
task.check() // task.check()
// task.deduplicate()
task.updateEsMapPoi()
// task.checkSpecialSymbols() // task.checkSpecialSymbols()
} }
@@ -395,6 +399,101 @@ class PoiWash {
} }
def deduplicate(){
//"三明市",
def citys = ["三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
def total = 0
def upindex = 0
for(def city : citys){
nameKeys = new HashSet()
def q = new BasicDBObject()
q.put("city", city)
q.put("status", new BasicDBObject('$ne', 2))
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
total++
def key = d.city +"##"+d.district+"##"+d.name+"##"+d.location.lon+"##"+d.location.lat
if(!nameKeys.contains(key)){
nameKeys << key
}else{
upindex++
def udata = new BasicDBObject()
udata.put("status", 2)
MAPPOI_CONN.update(new BasicDBObject('id',d.id),new BasicDBObject('$set', udata))
println "[${upindex}/${total}] Delete : "+d.city +" "+ d.district+" "+d.name
}
}
}
}
def updateEsMapPoi(){
// "三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市",
def citys = ["龙岩市","莆田市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
for(def city : citys) {
def datas = findPoiByQuery(city)
for(def d : datas){
updateMapPoi(d.id)
println city +" "+d.id
}
}
}
def updateMapPoi(def id){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/map_poi/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.status=2",
"lang": "painless"
},
"query": {
"term": {
"id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}
def findPoiByQuery(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("status", 2)
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
def rets = []
for (def d : dbcursor) {
rets << d
}
return rets;
}
def check(){ def check(){
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市", // "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
def citys = ["重庆市"] def citys = ["重庆市"]
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.* import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
Calendar calendar = Calendar.getInstance() Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate) calendar.setTime(startDate)
def count = 0 def count = 0
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate startDate = nextDate
} }
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301") ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, '202307')