@@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils {
|
||||
|
||||
public static void main(String[] args) {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
System.out.println(random(0, 10));
|
||||
System.out.println(random(-9, 9));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = CcbCommonUtils.citys;
|
||||
|
||||
for(def city : citys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
// for(def city : citys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
println citys.unique().size()
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
|
||||
+178
-2
@@ -1,15 +1,20 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
@@ -37,15 +42,74 @@ class DistrictDelete {
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
// task.run()
|
||||
// task.gzRun();
|
||||
// task.deleteDistrict()
|
||||
task.deleteCommPrice()
|
||||
// task.deleteCommPrice()
|
||||
// task.distHouseCountStat()
|
||||
task.addThirdPartyStat()
|
||||
// task.processRegion()
|
||||
}
|
||||
def processRegion(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d->
|
||||
if(d['B']){
|
||||
def region = RegionData.formatRegion("重庆",d['B'])
|
||||
if(region){
|
||||
d['B'] = region
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addThirdPartyStat(){
|
||||
def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1)
|
||||
def idMaps = [:]
|
||||
for(def d : datas){
|
||||
def id = d['A']
|
||||
idMaps.put(id, d['D'])
|
||||
}
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
|
||||
def totalCount = new AtomicLong()
|
||||
def map = findThirdPartyPrice("上海")
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def thirdLists = Collections.synchronizedList(new ArrayList());
|
||||
thirdLists << ["ID","区域","名称","总量"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def count = idMaps.get(d._id.toString())
|
||||
if (!count) {
|
||||
def dist = CommonDatautils.findDistrictById(d._id.toString())
|
||||
def priceMap = map.get(dist.region + "##" + dist.name)
|
||||
|
||||
if (priceMap) {
|
||||
thirdLists << [dist._id.toString(), dist.region, dist.name, priceMap.size().toString()]
|
||||
println dist._id.toString()+"@"+ dist.region+"@"+ dist.name+"@"+ priceMap.size()
|
||||
}
|
||||
}
|
||||
// println totalCount.get() + " " + d.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\上海第三方.xlsx",thirdLists, "房源")
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -60,6 +124,40 @@ class DistrictDelete {
|
||||
}
|
||||
}
|
||||
|
||||
def distHouseCountStat(){
|
||||
def city = "重庆"
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address"])
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def distCount = districts.size()
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
rets << ["ID","区域","名称","总量"]
|
||||
for(def i=0;i<districts.size();i++) {
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def count = findHouseCount(city, dist.region, dist.name, dist.alias)
|
||||
def count = findHouses(city, dist.region, "sell", 20210701,20210801, dist.name, dist.alias)
|
||||
println totalCount.get() +" "+city +" "+dist.region+" "+dist.name+" "+count
|
||||
if(count){
|
||||
rets << [dist._id.toString(), dist.region, dist.name, count]
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆7月份的房源统计.xlsx",rets, "房源")
|
||||
|
||||
}
|
||||
|
||||
def checkDist(city,dists){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
@@ -259,4 +357,82 @@ class DistrictDelete {
|
||||
sqlInstance.execute(sql)
|
||||
println "删除:"+tableName+" "+commid
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202112).append('$lte',202112))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.region+"##"+db.name
|
||||
def prices = map.get(key)
|
||||
if(!prices){
|
||||
prices = []
|
||||
map.put(key, prices)
|
||||
}
|
||||
if( db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
prices << db.zhugezhaofang_esf_price
|
||||
}
|
||||
if(db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
prices << db.anjuke_esf_price
|
||||
}
|
||||
if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
prices << db.get("58tc_esf_price")
|
||||
}
|
||||
if(db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
prices << db.soufang_esf_price
|
||||
}
|
||||
if(db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
prices << db.lianjia_esf_price
|
||||
}
|
||||
if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
prices << db.cityhouse_esf_price
|
||||
}
|
||||
|
||||
if(db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
prices << db.maitian_esf_price
|
||||
}
|
||||
|
||||
if(db.beike_esf_price && db.beike_esf_price>3000){
|
||||
prices << db.beike_esf_price
|
||||
}
|
||||
|
||||
if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
prices << db.get("365house_esf_price")
|
||||
}
|
||||
|
||||
if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
prices << db.zhongyuan_esf_price
|
||||
}
|
||||
|
||||
if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
prices << db.tuboshi_esf_price
|
||||
}
|
||||
|
||||
if( db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
prices << db.qfang_esf_price
|
||||
}
|
||||
|
||||
if(db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
prices << db.fangtan_esf_price
|
||||
}
|
||||
|
||||
if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
prices << db.fangduoduo_esf_price
|
||||
}
|
||||
|
||||
if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
prices << db.get("5i5j_esf_price")
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
|
||||
def prices = CcbCommonUtils.findGuidePrice(city, "2022-02-1", "2022-03-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+17
-18
@@ -22,25 +22,24 @@ class ExportOracleData {
|
||||
|
||||
def run() {
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
// //不会变
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()
|
||||
// exportBigdataDistrictData()
|
||||
exportEvaluationRateData()
|
||||
exportBigdataPoiData()
|
||||
exportBigdataDistrictData()
|
||||
//
|
||||
// exportGuidedPrice()//随人工改变
|
||||
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
//
|
||||
// //处理小区情况
|
||||
// exportBigdataCommunity()
|
||||
// exportCommunityStaging()
|
||||
// exportCommunityBuileType()
|
||||
exportGuidedPrice()//随人工改变
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
|
||||
// exportThirdPartyPrice2(lastMonth)
|
||||
// exportBigdataForsale("2021-12-19")
|
||||
// exportBigdataDeal("2021-12-01")
|
||||
// //处理小区情况
|
||||
exportBigdataCommunity()
|
||||
exportCommunityStaging()
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-01-19")
|
||||
exportBigdataDeal("2022-01-01")
|
||||
|
||||
|
||||
//计算结果相关
|
||||
@@ -56,11 +55,11 @@ class ExportOracleData {
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// def curDate = curMonth.toString()+"28"
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
//ok文件
|
||||
exportLastFile()
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
|
||||
|
||||
+7
-6
@@ -23,9 +23,10 @@ class LoadGuide {
|
||||
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
|
||||
|
||||
}
|
||||
def match(def filename){
|
||||
@@ -48,15 +49,15 @@ class LoadGuide {
|
||||
def datas = ExcelUtils.read(filePath,0, 1)
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
if(d['D']){
|
||||
def o = new BasicDBObject()
|
||||
o.city = d['E']
|
||||
o.city = d['D']
|
||||
o.region = d['A']
|
||||
o.name = d['B']
|
||||
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.match_city = d['D']
|
||||
o.match_region = d['E']
|
||||
o.match_name = d['F']
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
|
||||
+31
-26
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
}
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
|
||||
def testMap = findCaluateCityPPE(sqlInstance236, 202201)
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['C']){
|
||||
def oldcount = dealMap.get(d['C'])
|
||||
d['T'] = oldcount
|
||||
def casecount = testMap.get(d['C'])
|
||||
d['U'] = casecount
|
||||
println d
|
||||
}
|
||||
},0)
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
|
||||
//def testMap = findCaluateCityPPE(sqlInstance236, 202201)
|
||||
//ExcelUtils.modify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// def oldcount = dealMap.get(d['C'])
|
||||
// d['T'] = oldcount
|
||||
// def casecount = testMap.get(d['C'])
|
||||
// d['U'] = casecount
|
||||
// println d
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
|
||||
// if(d['I'] && d['K']){
|
||||
@@ -270,20 +270,25 @@ ExcelUtils.modify(fileName,{d->
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", new ObjectId(d['A']))
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// index++
|
||||
// println index
|
||||
// }
|
||||
//},0)
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d->
|
||||
def flag = d['M']
|
||||
if(flag==0.0){
|
||||
if(d['D']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(d['D']))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
index++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def region = d['B']
|
||||
|
||||
+25
-1
@@ -36,7 +36,8 @@ public class LianjiaCrawlByChrome {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.addDealRegion();
|
||||
task.standardRegion()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
@@ -105,6 +106,28 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
}
|
||||
|
||||
def standardRegion(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
if(d.region){
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
if(newRegion && newRegion!=d.region){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", d.city)
|
||||
query.put("region", d.region)
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", newRegion)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
@@ -132,6 +155,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
println m.city+ " "+m.name +" "+dist.region
|
||||
def ds = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user