第三方修改时间

This commit is contained in:
2022-01-19 16:38:42 +08:00
parent 31e1dea24c
commit 431ae4aa1b
5 changed files with 31 additions and 27 deletions
@@ -42,7 +42,7 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) { static main(args) {
@@ -24,36 +24,36 @@ class ExportOracleData {
} }
def run() { def run() {
def curMonth = 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// exportGuidedPrice()
// exportCommunityGuidedPrice(lastMonth.toString()+"28") //不会变
// exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
//处理小区情况
exportBigdataCommunity()
exportCommunityStaging()
exportCommunityBuileType()
// //exportThirdPartyPrice()//废弃掉,老表 // //exportThirdPartyPrice()//废弃掉,老表
// exportThirdPartyPrice2(lastMonth) // exportThirdPartyPrice2(lastMonth)
//
//
//
// exportBigdataForsale("2021-11-19") // exportBigdataForsale("2021-11-19")
//
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportCommunityStaging()
// exportCommunityBuileType()
// exportBigdataDistrictData()
// exportBigdataCommunity()
// exportBigdataDeal("2021-11-01")
// exportBigdataDeal("2021-11-01")
// exportAvmEvaluationGJNB(curMonth) // exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth)
//
// exportCityPrice(curMonth) // exportCityPrice(curMonth)
// exportRegionPrice(curMonth) // exportRegionPrice(curMonth)
// exportCommunityBasePriceGj(curMonth)//已合下面的合并 // exportCommunityBasePriceGj(curMonth)//已合下面的合并
exportCommunityBasePrice(curMonth) // exportCommunityBasePrice(curMonth)
// exportCommunityPdescribeGj(curMonth) // exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth) // exportCommCaseDetailGj(curMonth)
@@ -40,7 +40,7 @@ class StagingDistrictBatch extends BaseBatch{
results << headers results << headers
results.addAll(rows) results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据") //ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false); FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false);
} }
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch { class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202111//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202109 def startDate = 202110
def endDate = 202111 def endDate = 202112
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -35,6 +35,7 @@ public class LianjiaCrawlByChrome {
def task = new LianjiaCrawlByChrome(); def task = new LianjiaCrawlByChrome();
task.run() task.run()
// task.run1() // task.run1()
task.addDealRegion();
} }
def run1(){ def run1(){
@@ -100,14 +101,16 @@ public class LianjiaCrawlByChrome {
// getLogin(driver) // getLogin(driver)
// crawlData(driver, cityMap) // crawlData(driver, cityMap)
addDealRegion();
} }
def addDealRegion(){ def addDealRegion(){
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("region", null) q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q) def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = [] def keys = []
for(def d : dbcursor){ for(def d : dbcursor){
@@ -121,13 +124,14 @@ public class LianjiaCrawlByChrome {
def query = new BasicDBObject() def query = new BasicDBObject()
query.put("city", m.city) query.put("city", m.city)
query.put("region", null)
query.put("district_name", m.name) query.put("district_name", m.name)
query.put("source", "链家在线") // query.put("source", "链家在线")
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
def data = new BasicDBObject() def data = new BasicDBObject()
data.put("region", dist.region) data.put("region", dist.region)
def count = DEAL_PRICE_COLL.count(query) def ds = DEAL_PRICE_COLL.count(query)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
} }
} }