+11
-5
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳8月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -58,10 +58,16 @@ for(def d : datas){
|
||||
}catch(Exception e){
|
||||
println "room error"
|
||||
}
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
if(arrs[1].toString()){
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
}
|
||||
if(arrs.size()==4 && arrs[3].toString()){
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
}
|
||||
}else{
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
if(roomStr.toString()){
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -74,7 +80,7 @@ for(def d : datas){
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-08-01", "2023-08-31")
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -311,7 +311,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
def index = 0
|
||||
sqlInstance.withBatch(10000, sql){ ps->
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : newRows) {
|
||||
index++
|
||||
try{
|
||||
|
||||
+1
@@ -63,6 +63,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
citys = ['广州','重庆','南京','石家庄','廊坊','佛山','无锡','哈尔滨']
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-08-25", "2023-09-27")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-24")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+6
-6
@@ -40,8 +40,8 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -93,13 +93,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年7月总量","23年8月总量","23年9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-05-01"
|
||||
def date = "2023-06-01"
|
||||
def endDate = "2023-07-01"
|
||||
def newDate = "2023-08-01"
|
||||
def date = "2023-08-01"
|
||||
def endDate = "2023-09-01"
|
||||
def newDate = "2023-10-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+18
-3
@@ -1740,14 +1740,14 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPrice(city, region, name, weekend){
|
||||
public static def queryFirstDistrictPrice(city, region, name){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put("weekend", weekend);
|
||||
// queryCondition.put("weekend", weekend);
|
||||
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def dbo = null;
|
||||
while(cur.hasNext()) {
|
||||
@@ -2215,6 +2215,21 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryAllDistrictPrice(city, startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put('weekend', startDate);
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rMap = [:];
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
rMap.put(city + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
public static def queryAllDistrictPrice(startDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put('weekend', startDate);
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-08-01"
|
||||
static def statDate = "2023-09-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
task.esfParser("D:\\数据\\友达\\2023年9月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年10月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+122
-1
@@ -9,8 +9,19 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 价格对比,第三方,成交,建行价格,自已家价格
|
||||
*/
|
||||
class AddPriceTest {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
@@ -26,10 +37,120 @@ class AddPriceTest {
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
addFjwPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
exportPkPrice()
|
||||
// addRegion()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
d['C'] = dist.region
|
||||
})
|
||||
}
|
||||
def exportPkPrice(){
|
||||
//"南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安",
|
||||
def citys = ["宿迁"]
|
||||
|
||||
for(def c : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","alias"])
|
||||
println c+" district:"+districts.size()
|
||||
|
||||
def fjwPriceMap =[:]// CommonDatautils.queryAllDistrictPrice(c, DateUtils.parseFromTightByPattern("2023-10-08","yyyy-MM-dd"))
|
||||
def datas = addAllPrice(districts, fjwPriceMap)
|
||||
ExcelUtils.write("D:\\数据\\上海农总行\\江苏省价格比对.xlsx", datas, c)
|
||||
println c +" 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def addAllPrice(districts, fjwPriceMap){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
rets << ["id", "城市","区域","名称","建行价格","计算方式","可信度","房价网ToB价格","计算方式","可信度","近6个月成交均价"]
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createPriceData(cityCodeDataMap, dist, fjwPriceMap)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createPriceData(cityCodeDataMap, d, fjwPriceMap){
|
||||
def row = []
|
||||
row << d._id.toString()
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = findAvgPrice(d._id.toString(), cityId)
|
||||
if(map){
|
||||
row << map.avg_price
|
||||
row << map.calc_type
|
||||
row << map.credit_value
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
}
|
||||
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name)
|
||||
// def dbo = fjwPriceMap.get(d.city+"##"+d.region+"##"+d.name)
|
||||
if(dbo){
|
||||
row << dbo.avg_price
|
||||
if(dbo.value_type == 1){
|
||||
row << "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
row << "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
row << "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
row << "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
row << "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
row << "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
row << "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
row << "无方式"
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << dbo.credit
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
}
|
||||
|
||||
def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-09-01')
|
||||
if(tradePrice){
|
||||
row << tradePrice.toInteger()
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
|
||||
return row;
|
||||
}
|
||||
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
+101
-2
@@ -2,12 +2,14 @@ package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xingyebank.Data
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
@@ -30,14 +32,16 @@ class PoiWash {
|
||||
|
||||
public static def townsMap = [:]
|
||||
public static def regions = []
|
||||
public static def nameKeys = null
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiWash()
|
||||
// task.updateId()
|
||||
// task.run()
|
||||
// task.synch()
|
||||
task.check()
|
||||
|
||||
// task.check()
|
||||
// task.deduplicate()
|
||||
task.updateEsMapPoi()
|
||||
// task.checkSpecialSymbols()
|
||||
}
|
||||
|
||||
@@ -395,6 +399,101 @@ class PoiWash {
|
||||
}
|
||||
|
||||
|
||||
def deduplicate(){
|
||||
//"三明市",
|
||||
def citys = ["三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
|
||||
def total = 0
|
||||
def upindex = 0
|
||||
for(def city : citys){
|
||||
nameKeys = new HashSet()
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("status", new BasicDBObject('$ne', 2))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("id", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
total++
|
||||
def key = d.city +"##"+d.district+"##"+d.name+"##"+d.location.lon+"##"+d.location.lat
|
||||
if(!nameKeys.contains(key)){
|
||||
nameKeys << key
|
||||
}else{
|
||||
upindex++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 2)
|
||||
MAPPOI_CONN.update(new BasicDBObject('id',d.id),new BasicDBObject('$set', udata))
|
||||
println "[${upindex}/${total}] Delete : "+d.city +" "+ d.district+" "+d.name
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def updateEsMapPoi(){
|
||||
// "三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市",
|
||||
def citys = ["龙岩市","莆田市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
for(def city : citys) {
|
||||
def datas = findPoiByQuery(city)
|
||||
for(def d : datas){
|
||||
updateMapPoi(d.id)
|
||||
println city +" "+d.id
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/map_poi/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.status=2",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}
|
||||
|
||||
|
||||
def findPoiByQuery(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("status", 2)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("id", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for (def d : dbcursor) {
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def check(){
|
||||
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
|
||||
def citys = ["重庆市"]
|
||||
|
||||
+3
-3
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, '202307')
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user