导数据修改

This commit is contained in:
2022-02-22 17:02:25 +08:00
parent dd68bb1364
commit ec54ae056d
8 changed files with 263 additions and 58 deletions
@@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils {
public static void main(String[] args) {
for (int i = 0; i < 1000; i++) {
System.out.println(random(0, 10));
System.out.println(random(-9, 9));
}
}
}
@@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
for(def city : citys){
addDealPriceMatch(city);
}
// for(def city : citys){
// addDealPriceMatch(city);
// }
println citys.unique().size()
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -1,15 +1,20 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -37,15 +42,74 @@ class DistrictDelete {
dataSource.setProperties(connectionProperties)
}
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new DistrictDelete()
// task.run()
// task.gzRun();
// task.deleteDistrict()
task.deleteCommPrice()
// task.deleteCommPrice()
// task.distHouseCountStat()
task.addThirdPartyStat()
// task.processRegion()
}
def processRegion(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d->
if(d['B']){
def region = RegionData.formatRegion("重庆",d['B'])
if(region){
d['B'] = region
}
}
},0)
}
def addThirdPartyStat(){
def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1)
def idMaps = [:]
for(def d : datas){
def id = d['A']
idMaps.put(id, d['D'])
}
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
def totalCount = new AtomicLong()
def map = findThirdPartyPrice("上海")
final CountDownLatch latch = new CountDownLatch(dists.size())
def thirdLists = Collections.synchronizedList(new ArrayList());
thirdLists << ["ID","区域","名称","总量"]
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = idMaps.get(d._id.toString())
if (!count) {
def dist = CommonDatautils.findDistrictById(d._id.toString())
def priceMap = map.get(dist.region + "##" + dist.name)
if (priceMap) {
thirdLists << [dist._id.toString(), dist.region, dist.name, priceMap.size().toString()]
println dist._id.toString()+"@"+ dist.region+"@"+ dist.name+"@"+ priceMap.size()
}
}
// println totalCount.get() + " " + d.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\重庆农商行\\上海第三方.xlsx",thirdLists, "房源")
}
def run(){
@@ -60,6 +124,40 @@ class DistrictDelete {
}
}
def distHouseCountStat(){
def city = "重庆"
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def distCount = districts.size()
def rets = Collections.synchronizedList(new ArrayList());
rets << ["ID","区域","名称","总量"]
for(def i=0;i<districts.size();i++) {
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def count = findHouseCount(city, dist.region, dist.name, dist.alias)
def count = findHouses(city, dist.region, "sell", 20210701,20210801, dist.name, dist.alias)
println totalCount.get() +" "+city +" "+dist.region+" "+dist.name+" "+count
if(count){
rets << [dist._id.toString(), dist.region, dist.name, count]
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆7月份的房源统计.xlsx",rets, "房源")
}
def checkDist(city,dists){
def rets = Collections.synchronizedList(new ArrayList());
@@ -259,4 +357,82 @@ class DistrictDelete {
sqlInstance.execute(sql)
println "删除:"+tableName+" "+commid
}
def findThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city",city)
q.put("s_date", new BasicDBObject('$gte', 202112).append('$lte',202112))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.region+"##"+db.name
def prices = map.get(key)
if(!prices){
prices = []
map.put(key, prices)
}
if( db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
prices << db.zhugezhaofang_esf_price
}
if(db.anjuke_esf_price && db.anjuke_esf_price>3000){
prices << db.anjuke_esf_price
}
if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
prices << db.get("58tc_esf_price")
}
if(db.soufang_esf_price && db.soufang_esf_price>3000){
prices << db.soufang_esf_price
}
if(db.lianjia_esf_price && db.lianjia_esf_price>3000){
prices << db.lianjia_esf_price
}
if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
prices << db.cityhouse_esf_price
}
if(db.maitian_esf_price && db.maitian_esf_price>3000){
prices << db.maitian_esf_price
}
if(db.beike_esf_price && db.beike_esf_price>3000){
prices << db.beike_esf_price
}
if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
prices << db.get("365house_esf_price")
}
if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
prices << db.zhongyuan_esf_price
}
if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
prices << db.tuboshi_esf_price
}
if( db.qfang_esf_price && db.qfang_esf_price>3000){
prices << db.qfang_esf_price
}
if(db.fangtan_esf_price && db.fangtan_esf_price>3000){
prices << db.fangtan_esf_price
}
if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
prices << db.fangduoduo_esf_price
}
if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
prices << db.get("5i5j_esf_price")
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
}
return map
}
}
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
def prices = CcbCommonUtils.findGuidePrice(city, "2022-02-1", "2022-03-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -22,25 +22,24 @@ class ExportOracleData {
def run() {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// //不会变
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
//
// exportGuidedPrice()//随人工改变
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
//
// //处理小区情况
// exportBigdataCommunity()
// exportCommunityStaging()
// exportCommunityBuileType()
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
// exportThirdPartyPrice2(lastMonth)
// exportBigdataForsale("2021-12-19")
// exportBigdataDeal("2021-12-01")
// //处理小区情况
exportBigdataCommunity()
exportCommunityStaging()
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-01-19")
exportBigdataDeal("2022-01-01")
//计算结果相关
@@ -56,11 +55,11 @@ class ExportOracleData {
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// def curDate = curMonth.toString()+"28"
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//ok文件
exportLastFile()
// exportLastFile()
}
@@ -23,9 +23,10 @@ class LoadGuide {
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
}
def match(def filename){
@@ -48,15 +49,15 @@ class LoadGuide {
def datas = ExcelUtils.read(filePath,0, 1)
def rets = []
for(def d : datas){
if(d['E']){
if(d['D']){
def o = new BasicDBObject()
o.city = d['E']
o.city = d['D']
o.region = d['A']
o.name = d['B']
o.avg_price = d['C'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.match_city = d['D']
o.match_region = d['E']
o.match_name = d['F']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
}
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
def testMap = findCaluateCityPPE(sqlInstance236, 202201)
ExcelUtils.modify(fileName,{d->
if(d['C']){
def oldcount = dealMap.get(d['C'])
d['T'] = oldcount
def casecount = testMap.get(d['C'])
d['U'] = casecount
println d
}
},0)
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202201)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['T'] = oldcount
// def casecount = testMap.get(d['C'])
// d['U'] = casecount
// println d
// }
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -270,20 +270,25 @@ ExcelUtils.modify(fileName,{d->
// }
//},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d->
// if(d['A']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['A']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// }
//},0)
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d->
def flag = d['M']
if(flag==0.0){
if(d['D']){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(d['D']))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
index++
println index
}
}
},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
// if(d['A']){
// def region = d['B']
@@ -36,7 +36,8 @@ public class LianjiaCrawlByChrome {
def task = new LianjiaCrawlByChrome();
// task.run()
// task.run1()
task.addDealRegion();
// task.addDealRegion();
task.standardRegion()
}
def run1(){
@@ -105,6 +106,28 @@ public class LianjiaCrawlByChrome {
}
def standardRegion(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
if(d.region){
def newRegion = RegionData.formatRegion(d.city, d.region)
if(newRegion && newRegion!=d.region){
def query = new BasicDBObject()
query.put("city", d.city)
query.put("region", d.region)
def data = new BasicDBObject()
data.put("region", newRegion)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
}
}
def addDealRegion(){
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
@@ -132,6 +155,7 @@ public class LianjiaCrawlByChrome {
def data = new BasicDBObject()
data.put("region", dist.region)
println m.city+ " "+m.name +" "+dist.region
def ds = DEAL_PRICE_COLL.count(query)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}