Merge branch 'script_gj' into 'master'

7个城市

See merge request !738
This commit is contained in:
2023-10-23 20:47:56 +08:00
10 changed files with 266 additions and 24 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳8月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -58,10 +58,16 @@ for(def d : datas){
}catch(Exception e){
println "room error"
}
map.hall = arrs[1].toString().toInteger()
map.toilet = arrs[3].toString().toInteger()
if(arrs[1].toString()){
map.hall = arrs[1].toString().toInteger()
}
if(arrs.size()==4 && arrs[3].toString()){
map.toilet = arrs[3].toString().toInteger()
}
}else{
map.room = roomStr.toString().toDouble().toInteger()
if(roomStr.toString()){
map.room = roomStr.toString().toDouble().toInteger()
}
}
}
@@ -74,7 +80,7 @@ for(def d : datas){
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-08-01", "2023-08-31")
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
}
@@ -311,7 +311,7 @@ class BigdataCommunityBatch extends BaseBatch{
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def index = 0
sqlInstance.withBatch(10000, sql){ ps->
sqlInstance.withBatch(50000, sql){ ps->
for(def r : newRows) {
index++
try{
@@ -63,6 +63,7 @@ class BigdataDealBatch extends BaseBatch{
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
citys = ['广州','重庆','南京','石家庄','廊坊','佛山','无锡','哈尔滨']
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-08-25", "2023-09-27")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-24")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年7月总量","23年8月总量","23年9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-05-01"
def date = "2023-06-01"
def endDate = "2023-07-01"
def newDate = "2023-08-01"
def date = "2023-08-01"
def endDate = "2023-09-01"
def newDate = "2023-10-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -1740,14 +1740,14 @@ class CommonDatautils {
}
public static def queryDistrictPrice(city, region, name, weekend){
public static def queryFirstDistrictPrice(city, region, name){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
// queryCondition.put("weekend", weekend);
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", -1)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def dbo = null;
while(cur.hasNext()) {
@@ -2215,6 +2215,21 @@ class CommonDatautils {
}
public static def queryAllDistrictPrice(city, startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put('weekend', startDate);
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rMap = [:];
while(cur.hasNext()) {
DBObject dbo = cur.next()
rMap.put(city + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
}
return rMap;
}
public static def queryAllDistrictPrice(startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put('weekend', startDate);
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-08-01"
static def statDate = "2023-09-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
task.esfParser("D:\\数据\\友达\\2023年9月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年10月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -9,8 +9,19 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 价格对比,第三方,成交,建行价格,自已家价格
*/
class AddPriceTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -26,10 +37,120 @@ class AddPriceTest {
// addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
addFjwPrice(filePath)
// addFjwPrice(filePath)
exportPkPrice()
// addRegion()
}
def addRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
d['C'] = dist.region
})
}
def exportPkPrice(){
//"南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安",
def citys = ["宿迁"]
for(def c : citys){
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","alias"])
println c+" district:"+districts.size()
def fjwPriceMap =[:]// CommonDatautils.queryAllDistrictPrice(c, DateUtils.parseFromTightByPattern("2023-10-08","yyyy-MM-dd"))
def datas = addAllPrice(districts, fjwPriceMap)
ExcelUtils.write("D:\\数据\\上海农总行\\江苏省价格比对.xlsx", datas, c)
println c +" 完成"
}
}
def addAllPrice(districts, fjwPriceMap){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def rets = new CopyOnWriteArrayList()
rets << ["id", "城市","区域","名称","建行价格","计算方式","可信度","房价网ToB价格","计算方式","可信度","近6个月成交均价"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createPriceData(cityCodeDataMap, dist, fjwPriceMap)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createPriceData(cityCodeDataMap, d, fjwPriceMap){
def row = []
row << d._id.toString()
row << d.city
row << d.region
row << d.name
def cityId = cityCodeDataMap.get(d.city)
def map = findAvgPrice(d._id.toString(), cityId)
if(map){
row << map.avg_price
row << map.calc_type
row << map.credit_value
}else{
row << ""
row << ""
row << ""
}
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
// def dbo = fjwPriceMap.get(d.city+"##"+d.region+"##"+d.name)
if(dbo){
row << dbo.avg_price
if(dbo.value_type == 1){
row << "指导价"
}else if(dbo.value_type == 2){
row << "自身样本"
}else if(dbo.value_type == 3){
row << "第三方"
}else if(dbo.value_type == 4){
row << "交易数据"
}else if(dbo.value_type == 5){
row << "周边楼盘"
}else if(dbo.value_type == 6){
row << "推导"
}else if(dbo.value_type == 9){
row << "神经网络"
}else if(dbo.value_type == 10){
row << "无方式"
}else{
row << ""
}
row << dbo.credit
}else{
row << ""
row << ""
row << ""
}
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-09-01')
if(tradePrice){
row << tradePrice.toInteger()
}else{
row << ''
}
return row;
}
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -2,12 +2,14 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
@@ -30,14 +32,16 @@ class PoiWash {
public static def townsMap = [:]
public static def regions = []
public static def nameKeys = null
static main(args) {
def task = new PoiWash()
// task.updateId()
// task.run()
// task.synch()
task.check()
// task.check()
// task.deduplicate()
task.updateEsMapPoi()
// task.checkSpecialSymbols()
}
@@ -395,6 +399,101 @@ class PoiWash {
}
def deduplicate(){
//"三明市",
def citys = ["三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
def total = 0
def upindex = 0
for(def city : citys){
nameKeys = new HashSet()
def q = new BasicDBObject()
q.put("city", city)
q.put("status", new BasicDBObject('$ne', 2))
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
total++
def key = d.city +"##"+d.district+"##"+d.name+"##"+d.location.lon+"##"+d.location.lat
if(!nameKeys.contains(key)){
nameKeys << key
}else{
upindex++
def udata = new BasicDBObject()
udata.put("status", 2)
MAPPOI_CONN.update(new BasicDBObject('id',d.id),new BasicDBObject('$set', udata))
println "[${upindex}/${total}] Delete : "+d.city +" "+ d.district+" "+d.name
}
}
}
}
def updateEsMapPoi(){
// "三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市",
def citys = ["龙岩市","莆田市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
for(def city : citys) {
def datas = findPoiByQuery(city)
for(def d : datas){
updateMapPoi(d.id)
println city +" "+d.id
}
}
}
def updateMapPoi(def id){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/map_poi/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.status=2",
"lang": "painless"
},
"query": {
"term": {
"id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}
def findPoiByQuery(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("status", 2)
def keys = new BasicDBObject()
keys.put("id", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
def rets = []
for (def d : dbcursor) {
rets << d
}
return rets;
}
def check(){
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
def citys = ["重庆市"]
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, '202307')