From f5d39e53c49973d0bd63bf56c6fc89d49ba51be5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 4 Apr 2023 11:18:26 +0800 Subject: [PATCH 01/16] =?UTF-8?q?=E5=B8=B8=E5=B7=9E=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/FjwPricePkCcbPrice.groovy | 14 +- .../com/fangjia/script/gj/ZheShengBank.groovy | 40 ++ .../gj/crawl/AnjukeDistrictMatch.groovy | 412 ++++++++++++++++-- .../fangjia/script/gj/poc/AddPriceTest.groovy | 131 ++++++ .../fangjia/script/gj/poc/RentPrice.groovy | 84 ++++ .../script/jiangnanblank/CZDataBatch.groovy | 3 +- .../script/wenyan/temp/DongGuanBank.groovy | 6 +- 7 files changed, 654 insertions(+), 36 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/RentPrice.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy index dcc353655..4c9574fd8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/FjwPricePkCcbPrice.groovy @@ -33,7 +33,7 @@ class FjwPricePkCcbPrice { def task = new FjwPricePkCcbPrice() // task.run() // task.updatePrice() -// task.addFjwPrice() + task.addFjwPrice() // task.addCcbPrice() // task.addCcbPriceFromGuided() @@ -44,7 +44,7 @@ class FjwPricePkCcbPrice { // task.addCcbPrice2() // task.addCcbPrice3() - task.standRegion() +// task.standRegion() } def addCcbPrice2(){ @@ -202,13 +202,13 @@ class FjwPricePkCcbPrice { } def addFjwPrice(){ - ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d-> - if(!d['M']){ - def dist = CommonDatautils.findDistrictById(d['D']) + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d-> + if(!d['K']){ + def dist = CommonDatautils.findDistrictById(d['C']) if(dist){ - def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd")) + def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd")) if(dbo){ - d['M'] = dbo.avg_price + d['K'] = dbo.avg_price } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy new file mode 100644 index 000000000..2817d78ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ZheShengBank.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.gj + +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.bson.Document + +import static com.mongodb.client.model.Filters.* + +MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def startDate = DateUtil.parseDate("20210301", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230401", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def data = [[ "公司名称","访问链接", "访问时间", "总量"]] +def rMap = [:] +def index = 0 +OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc -> + index++ + def day = doc.c_date.format("yyyy-MM-dd") + def count = rMap.get(day) + if (!count) { + rMap.put(day, 1) + } else { + rMap.put(day, count + 1) + } + if(index%10000==0){ + println index+" "+new Date().toString() + } +} + +rMap.each {k,val-> + println k+" "+val +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy index 7362ebb6a..21816be71 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.crawl +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils import com.fangjia.script.linxsh.util.AddressUtil @@ -11,26 +12,67 @@ import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBList import com.mongodb.BasicDBObject import groovy.sql.Sql +import org.apache.commons.lang.StringUtils import org.apache.lucene.index.Term import org.apache.lucene.search.BooleanClause import org.apache.lucene.search.BooleanQuery import org.apache.lucene.search.TermQuery import org.bson.types.ObjectId +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + class AnjukeDistrictMatch { + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", "postgres", "postgres", "org.postgresql.Driver") static main(args) { def task = new AnjukeDistrictMatch(); // task.run(); - task.matchCommunity() +// task.runCsv(); +// task.updateCredit() +// task.matchCommunity()//查询es小区数据 // task.run1() -// task.updateField() +// task.updateGreenRateField()//根据第三方数据填补库中字段 +// task.updteGreenRateFieldByCsv() // task.fenQiRun() -// task.findAliasDatas("淮安","水沐未来印象花园") +// task.findAliasDatas("淮安","水沐未来印象花园")//模糊查找Es别名数据 +// task.repeatDistByAlias()//根据第三方网站别名查找重复数据 + + task.matchCommunityByCsv() + } + + def updateCredit(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv") + def results = [] + for(def i=0;i if(row['Q'] && !row['Q'].contains("接口")){ @@ -155,18 +197,242 @@ class AnjukeDistrictMatch { }) } + + def matchCommunityByCsv(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-1.csv") + def map = [:] + for(def i=0;i2){ + row << arrs[2] + } + if(arrs.size()>3){ + row << arrs[3] + } + if(arrs.size()>4){ + row << arrs[4] + } + } + } + lastRets << row + } + batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站建筑面积(匹配).csv",null,"GBK") + } + + def updteGreenRateFieldByCsv(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\测试.csv") + for(def i=0;i=0.9))){ + def dist = CommonDatautils.findSeaweedById(row[25], ["virescence_percent"]) + if(dist && !dist.virescence_percent){ + def val = row[12] + + if(!val){ + val = row[13] + } + if(!val){ + val = row[14] + } + if(!val){ + val = row[11] + } + + if(!val){ + val = row[21] + } + if(!val){ + val = row[22] + } + + if(!val){ + val = row[6] + } + if(!val){ + val = row[10] + } + + if(!val){ + val = row[9] + } + if(!val){ + val = row[5] + } + + if(!val){ + val = row[16] + } + + if(!val){ + val = row[15] + } + + if(!val){ + val = row[23] + } + if(!val){ + val = row[19] + } + if(!val){ + val = row[17] + } + if(!val){ + val = row[8] + } + + if(!val){ + val = row[18] + } + if(!val){ + val = row[20] + } + + if(!val){ + val = row[3] + } + + if(!val){ + val = row[4] + } + + if(!val){ + val = row[7] + } + + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("virescence_percent", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + println row[25] +" "+val.toString() + } + } + } + } + + } + // def run1(){ // def regionPointsMap = findRegionRange("重庆") // def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) // } + + def runCsv(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")//第三方网站绿化率 (3).csv + def results = batchMatch(datas) + batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv",null,"GBK") + } + + + def batchMatch(datas){ + def results = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i + index++ if(row['A'] && row['B']){ def city = row['A'] - def name = row['B'] - def region = null + def name = row['C'] + def region = row['B'] + + if(name && name.contains("·")){ + name = name.replace("·", "") + } def flag = "名称匹配" def dist = matchDist(city, region, name) @@ -203,26 +469,26 @@ class AnjukeDistrictMatch { }else{ def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null) if(jsonObject != null && jsonObject.precise!=-1) { - row['F'] = "匹配接口" - row['G'] = jsonObject.get("id") - row['D'] = jsonObject.get("district") - row['C'] = jsonObject.get("name") + row['Y'] = "匹配接口" + row['Z'] = jsonObject.get("id") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") + row['AC'] = jsonObject.get("credit") } } - - if(row['G']){ - def dbo = CommonDatautils.findDistrictById(row['G']) - if(dbo){ - row['H'] = dbo.estate_type2 - row['I'] = dbo.name_cat - - def count = findBigdataCommunity(row['G']) - if(count){ - row['E'] = "是" - } - } - } - + println index +" "+row['Y'] +// if(row['G']){ +// def dbo = CommonDatautils.findDistrictById(row['G']) +// if(dbo){ +// row['H'] = dbo.estate_type2 +// row['I'] = dbo.name_cat +// +// def count = findBigdataCommunity(row['G']) +// if(count){ +// row['E'] = "是" +// } +// } +// } } }) } @@ -286,6 +552,57 @@ class AnjukeDistrictMatch { return rets; } + def repeatDistByAlias(){ + def rets = findAliasDist() +// def datas = [] +// for(def r : rets){ +// datas << [r.city, r.region, r.name, r.alias] +// } +// ExcelUtils.write("D:\\数据\\小区\\第三方有别名的小区.xlsx", datas, "test") + def districts = CommonDatautils.findAllDistrictByCity("常州", "district", ["city","region","name"]) + def distKeys = [] + for(def d : districts){ + distKeys << d.region+"#"+d.name + } + + def results = [] + for(def t : rets){ + def names = [] + names << t.region + "#" +t.name + def arrs = t.alias.split(",|,") + for(def a : arrs){ + names << t.region + "#" + a + if(a.contains("(")){ + names << t.region + "#" + a.replace("(","").replace(")", "") + } + } + + def set = new HashSet() + for(def n : names){ + if(distKeys.contains(n)){ + set << n + } + } + + if(set.size()>1){ + results << [t.city, t.region, t.name, set.join(" "),t.url] + } + } + ExcelUtils.write("D:\\数据\\小区\\第三方别名验证重复小区.xlsx", results, "test") + } + + def findAliasDist(){ + def rets = [] + def sql = "select * from community where city='常州'" + ElasticUtils.executeSqlQuery(sql, { d-> + if(d.alias){ + rets << ["city":d.city, "region":d.region, "name":d.name, "source":d.source, "alias":d.alias, "url":d.url] + } + + }) + return rets; + } + def findAliasDatas(city, name){ def must = [] def term1 = ["match":["city":city]] @@ -309,4 +626,49 @@ class AnjukeDistrictMatch { } return rets; } + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split(",",-1) + rows << row + } + return rows; + } + + def batchCreateFile(ret, fileName, headers, charset){ + OutputStream out = new FileOutputStream(fileName); + def sb = new StringBuffer(); +// sb.append(headers.join("@|@")) +// sb.append(System.getProperty("line.separator")); + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + index++ + sb.append(StringUtils.join(row, ",")) + sb.append(System.getProperty("line.separator")); + + println index + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy new file mode 100644 index 000000000..b994fde09 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -0,0 +1,131 @@ +package com.fangjia.script.gj.poc + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +class AddPriceTest { + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com") + + static main(args) { + def task = new AddPriceTest(); + task.run(); + } + def run(){ + def filePath = "D:\\数据\\招商\\招商.xlsx" +// addGXDPrice(filePath) +// addCcbPrice(filePath) + addThirdPrice(filePath) + + } + + + def addCcbPrice(filepath){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify(filepath,{ d-> + if(d['M']){ + def cityId = cityCodeDataMap.get(d['N']) + def map = findAvgPrice(d['M'], cityId) + if(map){ + d['AK'] = map.avg_price + d['AL'] = map.calc_type + d['AM'] = map.credit_value + } + } + }) + } + + + def findAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + + + def addThirdPrice(filepath){ + ExcelUtils.multiThreadingModify(filepath, { d -> + if (d['M']) { + def dist = CommonDatautils.findDistrictById(d['M']) + if(dist){ + def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) + d['AN'] = thirdPrice + } + } + }) + } + + + def findThirdPartyPrice(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302)) + + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def price = null + for(def db : dbcursor){ + if(!price && db.lianjia_esf_price && db.lianjia_esf_price>2000){ + price = db.lianjia_esf_price + break; + } + + if(!price && db.beike_esf_price && db.beike_esf_price>2000){ + price = db.beike_esf_price + break + } + } + return price + } + + + def addGXDPrice(filepath){ + ExcelUtils.multiThreadingModify(filepath, { d -> + if (d['M']) { + def dist = CommonDatautils.findDistrictById(d['M']) + if(dist){ + def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) + d['AO'] = gxdPrice + } + } + }) + } + + def findGXDPrice(def city, region, name){ + + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("source", "GXD") + def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1) + def price = null + for(def d : dbcursor){ + price = d.price + } + return price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/RentPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/RentPrice.groovy new file mode 100644 index 000000000..73a7cf8b9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/RentPrice.groovy @@ -0,0 +1,84 @@ +package com.fangjia.script.gj.poc + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +class RentPrice { + + static main(args) { + def task = new RentPrice(); + task.run("D:\\数据\\招商\\楼盘POC测试样例地址.xlsx"); + } + def run(filepath){ + def index = 0 + ExcelUtils.multiThreadingModify(filepath, { d -> + index++ + if(d['M']){ + def dist = CommonDatautils.findDistrictById(d['M']) + if(dist){ + d['AZ'] = findHouses(dist.city, dist.region, "rent", 20221201,20230330, dist.name, dist.alias) + } + println index + " "+ d['AZ'] + } + + }) + } + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + +// BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); +// inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com")); +// inBuilder.should(QueryBuilders.matchQuery("root", "ke.com")); +// queryBuilder.must(inBuilder) + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def prices = [] + println city + " "+name+" "+houses.size() + for(def d : houses){ + prices << d.avg_price + } + + if(prices && prices.size()>=3){ + return (int)(prices.sum()/prices.size()) + } + return null + } + + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 289d83af3..d55d98dc4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -32,8 +32,9 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class CZDataBatch extends BaseBatch{ - + public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"] + public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl) //final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls"); //final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy index 520cf9f99..b1a045e3f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230301", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230401", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -38,6 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301") +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303") From fa7e1145bd84cbf74a7b6282f3ec7285ebbcb027 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 4 Apr 2023 16:34:39 +0800 Subject: [PATCH 02/16] =?UTF-8?q?=E5=88=AB=E5=90=8D=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/crawl/AnjukeDistrictMatch.groovy | 136 +++++++++++++++++- .../script/jiangnanblank/FzDataUtils.groovy | 8 +- 2 files changed, 140 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy index 21816be71..8ba85525b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -46,7 +46,9 @@ class AnjukeDistrictMatch { // task.findAliasDatas("淮安","水沐未来印象花园")//模糊查找Es别名数据 // task.repeatDistByAlias()//根据第三方网站别名查找重复数据 - task.matchCommunityByCsv() +// task.matchCommunityByCsv() +// task.updteBuildAreaFieldByCsv() + task.updadteAllAreaFieldByCsv() } def updateCredit(){ @@ -207,7 +209,7 @@ class AnjukeDistrictMatch { map.put(row[0]+"#"+row[1]+"#"+row[2], row[24]+"#"+row[25]+"#"+row[26]+"#"+row[27]+"#"+row[28]) } - def datas1 = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站建筑面积.csv") + def datas1 = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站占地面积.csv") def lastRets = [] for(def i=0;i=0.9))){ + def dist = CommonDatautils.findSeaweedById(row[17], ["build_area"]) + if(dist && !dist.build_area){ + def val = row[8] + + if(!val){ + val = row[7] + } + if(!val){ + val = row[9] + } + if(!val){ + val = row[14] + } + + if(!val){ + val = row[15] + } + if(!val){ + val = row[12] + } + + if(!val){ + val = row[5] + } + if(!val){ + val = row[6] + } + + if(!val){ + val = row[3] + } + if(!val){ + val = row[13] + } + + if(!val){ + val = row[4] + } + + if(!val){ + val = row[10] + } + + if(!val){ + val = row[11] + } + + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("build_area", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + println row[17] +" "+val.toString() + } + } + } + } + } + + def updadteAllAreaFieldByCsv(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站占地面积(匹配).csv") + for(def i=0;i=0.9))){ + def dist = CommonDatautils.findSeaweedById(row[13], ["all_area"]) + if(dist && !dist.all_area){ + def val = row[3] + + if(!val){ + val = row[4] + } + if(!val){ + val = row[5] + } + + if(!val){ + val = row[8] + } + + if(!val){ + val = row[9] + } + if(!val){ + val = row[10] + } + + if(!val){ + val = row[11] + } + if(!val){ + val = row[6] + } + + if(!val){ + val = row[7] + } + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("all_area", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + println row[13] +" "+val.toString() + } + } + } + } + } + // def run1(){ // def regionPointsMap = findRegionRange("重庆") // def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy index bcc75b6a0..8a9325c9b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy @@ -31,7 +31,13 @@ class FzDataUtils { map.cat = "district"; // 楼盘名称 map.name = dist.name - map.alias = dist.alias + def alias = dist.alias + if(alias){ + if(alias.size()>5){ + alias = alias[0..4] + } + map.alias = alias + } map.build_date = getBuildDate(dist); //附属属性 map.build_type = dist.build_type From 402fc0641addd5a814abb335aa72948edac1eddc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 6 Apr 2023 17:23:33 +0800 Subject: [PATCH 03/16] update --- .../com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 2e14d2f29..96ed55fcb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch { } def calcMonthlyPrice(){ - def citys = ["福州"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) @@ -75,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch { index++ def districts = [] if(city == "福州"){ - districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("name","翔鹰山庄"), keys).toList() + districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() }else{ districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) } From f6d308359b76870ceb2a230204fee10925088366 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 7 Apr 2023 11:11:57 +0800 Subject: [PATCH 04/16] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E8=A1=8C=E6=83=85?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=97=E6=AE=B5=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCitypriceBatch.groovy | 349 +++++++++++++++++- .../gj/crawl/AnjukeDistrictMatch.groovy | 195 +++++++++- 2 files changed, 524 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 9a7d539e9..0d33825c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -28,28 +28,45 @@ class BigdataCitypriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + def curRunDate = null def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") + def oracleSqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + public static def cityCodeMap = CcbCommonUtils.findCityCodeValue() + static main(args) { runBatch(new BigdataCitypriceBatch(), args) } @Override void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) - def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); - def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = new Date() + def citys = CcbCommonUtils.citys_all //CcbCommonUtils.getRunningCitys(args) + if(args){ + def dateStr = args[0] + curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01 + } + + println "args 参数:"+args.toString() +" 城市数量:"+citys.size()+" curDate: "+ curDate + + curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger(); + def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger(); + def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(curDate,-1), "yyyyMMdd").toInteger(); + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) + + def lastYearDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-12), "yyyyMMdd").toInteger(); cityRun(citys, startDate, endDate, cityCodeDataMap) regionRun(citys, startDate, endDate, cityCodeDataMap) - districtRun(citys, startDate, endDate, cityCodeDataMap) + districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap) } - def cityRun(citys, startDate, endDate, cityCodeDataMap){ + def cityRun(citys, startDate, endDate, dealStartDate, cityCodeDataMap){ def rets = [] def index = 0 for(def city : citys){ @@ -177,11 +194,18 @@ class BigdataCitypriceBatch extends BaseBatch{ println "end: "+new Date() } - def districtRun(citys, startDate, endDate, cityCodeDataMap){ + def districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap){ def rets = [] def allDistricts = [] def allPriceMap = [:] def allRentMap = [:] + def allDealCountMap = [:] + def allCityRankMap = [:] + def allRegionRankMap = [:] + def allDistCityPreMap = [:] + def allDistRegionPreMap = [:] + def allLastMonthMap = [:] + def allLastYearMap = [:] for(def city : citys) { def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]); allDistricts.addAll(districts) @@ -192,9 +216,36 @@ class BigdataCitypriceBatch extends BaseBatch{ def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate) allRentMap = allRentMap + rentMap + def cityCode = cityCodeMap.get(city) + def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate) + allDealCountMap = allDealCountMap + dealCountMap + + + def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6)) + + def cityRankMap = findCommunityCityRank(datas) + allCityRankMap = allCityRankMap + cityRankMap + + def regionRankMap = findCommunityRegionRank(datas) + allRegionRankMap = allRegionRankMap + regionRankMap + + def cityPrice = findCcbCityPrice(cityCode, startDate.toString().substring(0,6)) + def regionPriceMap = findCcbRegionPrice(cityCode, startDate.toString().substring(0,6)) + + def cityPreMap = districtCityPrictRate(cityPrice, datas) + allDistCityPreMap = allDistCityPreMap + cityPreMap + + def regionPreMap = districtRegionPriceRate(regionPriceMap, datas) + allDistRegionPreMap = allDistRegionPreMap + regionPreMap + + def lastMonthMap = findDistQuotations(cityCode, dealStartDate.replace("-","").substring(0,6)) + allLastMonthMap = allLastMonthMap + lastMonthMap + + def lastYearMonthMap = findDistQuotations(cityCode, lastYearDate.toString().substring(0,6)) + allLastYearMap = allLastYearMap + lastYearMonthMap } - def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap) + def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap) if(cityLists){ rets.addAll(cityLists) } @@ -212,7 +263,7 @@ class BigdataCitypriceBatch extends BaseBatch{ - def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ + def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){ def totalCount = new AtomicLong() def distCount = districts.size() final CountDownLatch latch = new CountDownLatch(distCount) @@ -224,7 +275,7 @@ class BigdataCitypriceBatch extends BaseBatch{ void run() { try { totalCount.addAndGet(1) - def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap) + def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap) if(row){ rets << row println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6] @@ -271,7 +322,7 @@ class BigdataCitypriceBatch extends BaseBatch{ } - def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ + def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){ def sellDataType = 1 def rentDataType = 1 def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2) @@ -400,6 +451,155 @@ class BigdataCitypriceBatch extends BaseBatch{ row << 1 row << 'FJW' + + //成交样本量,成交样本量同比, 成交样本量环比 + if(dealCountMap){ + def dealCount = dealCountMap.get(dist._id.toString()) + row << dealCount?dealCount:'' + def dealMarkup = '' + def dealYoy = '' + if(allLastMonthMap){ + def dealMarkupMap = allLastMonthMap.get(dist._id.toString()) + if(dealMarkupMap && dealMarkupMap.deal_count){ + def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count + dealMarkup = DataHandleUtils.getDoubleRound(2, markup) + } + } + if(allLastYearMap){ + def dealYoyMap = allLastYearMap.get(dist._id.toString()) + if(dealYoyMap && dealYoyMap.deal_count){ + def markup = (dealCount-dealYoyMap.deal_count)*100/dealYoyMap.deal_count + dealYoy = DataHandleUtils.getDoubleRound(2, markup) + } + } + row << dealYoy + row << dealMarkup + + }else{ + row << '' + row << '' + row << '' + } + + //出售平均单价的同比,环比 + if(sell_avg_price){ + def markup = '' + if(allLastMonthMap){ + def dataMap = allLastMonthMap.get(dist._id.toString()) + if(dataMap && dataMap.saleprice){ + markup = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice + markup = DataHandleUtils.getDoubleRound(2, markup) + } + } + def yoy = '' + if(allLastYearMap){ + def dataMap = allLastYearMap.get(dist._id.toString()) + if(dataMap && dataMap.saleprice){ + yoy = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice + yoy = DataHandleUtils.getDoubleRound(2, markup) + } + } + row << yoy + row << markup + }else{ + row << '' + row << '' + } + + //出售样本量同比,环比 + if(sell_count){ + def markup = '' + if(allLastMonthMap){ + def dataMap = allLastMonthMap.get(dist._id.toString()) + if(dataMap && dataMap.salecount){ + markup = (sell_count - dataMap.salecount)*100/dataMap.salecount + markup = DataHandleUtils.getDoubleRound(2, markup) + } + } + def yoy = '' + if(allLastYearMap){ + def dataMap = allLastYearMap.get(dist._id.toString()) + if(dataMap && dataMap.salecount){ + yoy = (sell_count - dataMap.salecount)*100/dataMap.salecount + yoy = DataHandleUtils.getDoubleRound(2, markup) + } + } + row << yoy + row << markup + }else{ + row << '' + row << '' + } + + //出租房源量同比,环比 + if(rent_count){ + def markup = '' + if(allLastMonthMap){ + def dataMap = allLastMonthMap.get(dist._id.toString()) + if(dataMap && dataMap.salecount && dataMap.rentcount>0){ + markup = (rent_count - dataMap.rentcount)*100/dataMap.rentcount + markup = DataHandleUtils.getDoubleRound(2, markup) + } + } + def yoy = '' + if(allLastYearMap){ + def dataMap = allLastYearMap.get(dist._id.toString()) + if(dataMap && dataMap.salecount && dataMap.rentcount>0){ + yoy = (rent_count - dataMap.rentcount)*100/dataMap.rentcount + yoy = DataHandleUtils.getDoubleRound(2, markup) + } + } + row << yoy + row << markup + }else{ + row << '' + row << '' + } + //小区价格城市排名 + if(allCityRankMap){ + def cityRank = allCityRankMap.get(dist._id.toString()) + if(cityRank){ + row << cityRank + }else{ + row << '' + } + }else{ + row << '' + } + //小区价格区域排名 + if(allRegionRankMap){ + def regionRank = allRegionRankMap.get(dist._id.toString()) + if(regionRank){ + row << regionRank + }else{ + row << '' + } + }else{ + row << '' + } + //高于区县平均价比率 + if(allDistRegionPreMap){ + def regionRadio = allDistRegionPreMap.get(dist._id.toString()) + if(regionRadio){ + row << regionRadio + }else{ + row << '' + } + }else{ + row << '' + } + + //高于城市平均价比率 + if(allDistCityPreMap){ + def cityRadio = allDistCityPreMap.get(dist._id.toString()) + if(cityRadio){ + row << cityRadio + }else{ + row << '' + } + }else{ + row << '' + } return row } return null @@ -723,6 +923,131 @@ class BigdataCitypriceBatch extends BaseBatch{ } + def findDealDatesByCity(cityCode, startDate, endDate){ + def sql = "SELECT gujia_comm_id,count(1) from base.t0831_bigdata_deal_gj_h WHERE urbn_id='${cityCode}' and txn_dt>='${startDate}' and txn_dt<'${endDate}' GROUP BY gujia_comm_id"; + def rMap = [:] + pgsqlInstance.eachRow(sql, { r -> + rMap.put(r.gujia_comm_id, r.count) + }) + return rMap; + } + + + def findCommunityCityRank(datas){ + def values = [] + for(def m : datas){ + values << m.avg_price + } + values = values.sort().reverse() + + def resMap = [:] + def total = values.size() + for(def m : datas){ +// def index = 0 +// for(def v : values){ +// index++ +// if(m.avg_price == v){ +// break; +// } +// } + def index = values.indexOf(m.avg_price)+1 + resMap.put(m.comm_id, index+"|"+total) + } + return resMap + } + + def findCommunityRegionRank(datas){ + + def regionMap = [:] + for(def m : datas){ + def prices = regionMap.get(m.region) + if(!prices){ + prices = [] + regionMap.put(m.region, prices) + } + prices << m.avg_price + prices.sort().reverse() + } + + + def resMap = [:] + for(def m : datas){ + def prices = regionMap.get(m.region) + def total = prices.size() + def index = prices.indexOf(m.avg_price)+1 +// for(def v : prices){ +// index++ +// if(m.avg_price == v){ +// break; +// } +// } + resMap.put(m.comm_id, index+"|"+total) + } + return resMap + } + + def findCommunityPriceByCity(cityCode, month){ + def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def datas = [] + pgsqlInstance.eachRow(sql, { r -> + datas << ["comm_id":r.gujia_bds_id, "avg_price":r.gujia_bds_clc_avprc, "region":r.gujia_admndiv_nm] + }) + return datas; + } + + def findCcbCityPrice(cityCode, month){ + def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def avgPrice = null + pgsqlInstance.eachRow(sql, { r -> + avgPrice = r.gujia_bds_clc_avprc + }) + return avgPrice; + } + + + + def findCcbRegionPrice(cityCode, month){ + def sql = "SELECT gxda_admndiv_nm,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def regionPriceMap = [:] + pgsqlInstance.eachRow(sql, { r -> + regionPriceMap.put(r.gxda_admndiv_nm, r.gujia_bds_clc_avprc) + }) + return regionPriceMap; + } + + + def districtCityPrictRate(cityPrice, datas){ + def distCityRateMap = [:] + if(cityPrice){ + for(def m : datas){ + def val = (m.avg_price-cityPrice)*100/cityPrice + distCityRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val)) + } + } + return distCityRateMap; + } + + + def districtRegionPriceRate(regionPriceMap, datas){ + def distRegionRateMap = [:] + if(regionPriceMap){ + for(def m : datas){ + def regionAvgPrice = regionPriceMap.get(m.region) + def val = (m.avg_price-regionAvgPrice)*100/regionAvgPrice + distRegionRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val)) + } + } + return distRegionRateMap; + } + + def findDistQuotations(cityCode, month){ + def sql = "select commid, deal_count, saleprice,salecount,rentcount from bigdata_commprice_fjw where ymid=${month} and cityid=${cityCode}"; + def markupMap = [:] + oracleSqlInstance.eachRow(sql, { r -> + markupMap.put(r.commid, ["deal_count":r.DEAL_COUNT, "saleprice":r.SALEPRICE,"salecount":r.SALECOUNT,"rentcount":r.RENTCOUNT]) + }) + return markupMap; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy index 8ba85525b..b28cc2770 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -48,7 +48,9 @@ class AnjukeDistrictMatch { // task.matchCommunityByCsv() // task.updteBuildAreaFieldByCsv() - task.updadteAllAreaFieldByCsv() +// task.updadteAllAreaFieldByCsv() +// task.updatePlotRateFieldByCsv() + task.updateFinishDateFieldByCsv() } def updateCredit(){ @@ -201,7 +203,7 @@ class AnjukeDistrictMatch { def matchCommunityByCsv(){ - def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-1.csv") + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv") def map = [:] for(def i=0;i1){ + row << arrs[1] + } if(arrs.size()>2){ row << arrs[2] } @@ -233,7 +237,7 @@ class AnjukeDistrictMatch { } lastRets << row } - batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站占地面积(匹配).csv",null,"GBK") + batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站竣工时间(匹配).csv",null,"GBK") } def updteGreenRateFieldByCsv(){ @@ -465,15 +469,190 @@ class AnjukeDistrictMatch { } } + + def updatePlotRateFieldByCsv(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv") + for(def i=0;i=0.9))){ + def dist = CommonDatautils.findSeaweedById(row[23], ["cubage_percent"]) + if(dist && !dist.cubage_percent){ + def val = row[10] + + if(!val){ + val = row[11] + } + if(!val){ + val = row[12] + } + + if(!val){ + val = row[19] + } + if(!val){ + val = row[20] + } + if(!val){ + val = row[21] + } + + if(!val){ + val = row[6] + } + if(!val){ + val = row[8] + } + + if(!val){ + val = row[9] + } + if(!val){ + val = row[13] + } + if(!val){ + val = row[14] + } + + if(!val){ + val = row[17] + } + + if(!val){ + val = row[3] + } + if(!val){ + val = row[4] + } + if(!val){ + val = row[5] + } + + if(!val){ + val = row[15] + } + if(!val){ + val = row[16] + } + + if(!val){ + val = row[18] + } + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("cubage_percent", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + println row[23] +" "+val.toString() + } + } + } + } + } + // def run1(){ // def regionPointsMap = findRegionRange("重庆") // def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) // } + def updateFinishDateFieldByCsv(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\竣工时间.csv") + for(def i=0;i=0.9))){ + def dist = CommonDatautils.findSeaweedById(row[23], ["finish_building_date"]) + if(dist && !dist.finish_building_date){ + def val = row[10] + + if(!val){ + val = row[11] + } + if(!val){ + val = row[12] + } + + if(!val){ + val = row[19] + } + if(!val){ + val = row[20] + } + if(!val){ + val = row[21] + } + + if(!val){ + val = row[6] + } + if(!val){ + val = row[8] + } + + if(!val){ + val = row[9] + } + if(!val){ + val = row[13] + } + if(!val){ + val = row[14] + } + + if(!val){ + val = row[17] + } + + if(!val){ + val = row[3] + } + if(!val){ + val = row[4] + } + if(!val){ + val = row[5] + } + + if(!val){ + val = row[15] + } + if(!val){ + val = row[16] + } + + if(!val){ + val = row[18] + } + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("finish_building_date", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + println row[23] +" "+val.toString() + } + } + } + } + } + def runCsv(){ - def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")//第三方网站绿化率 (3).csv + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配.csv")//第三方网站绿化率 (3).csv def results = batchMatch(datas) - batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv",null,"GBK") + batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配-2.csv",null,"GBK") } @@ -488,7 +667,7 @@ class AnjukeDistrictMatch { void run() { try{ totalCount.addAndGet(1) - if (row[0] && row[1]) { + if (row[0] && row[1] && !row[22]) { def city = row[0] def region = row[1] def name = row[2] From 04438ee9c24e54e3b62b0c8378b73dfc9e7bdaae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 7 Apr 2023 11:38:57 +0800 Subject: [PATCH 05/16] update --- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 0d33825c8..087d0ee8d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -66,7 +66,7 @@ class BigdataCitypriceBatch extends BaseBatch{ districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap) } - def cityRun(citys, startDate, endDate, dealStartDate, cityCodeDataMap){ + def cityRun(citys, startDate, endDate, cityCodeDataMap){ def rets = [] def index = 0 for(def city : citys){ From a25d31f1488cd57d931b085384077850d07febcf Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 7 Apr 2023 12:02:12 +0800 Subject: [PATCH 06/16] =?UTF-8?q?=E5=8C=BA=E5=9F=9F=E5=9D=87=E4=BB=B7?= =?UTF-8?q?=E4=B8=BA=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 087d0ee8d..726039ce9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -46,7 +46,7 @@ class BigdataCitypriceBatch extends BaseBatch{ void process(Object args) { def curDate = new Date() - def citys = CcbCommonUtils.citys_all //CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.citys_all //CcbCommonUtils.getRunningCitys(args) if(args){ def dateStr = args[0] curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01 @@ -1033,8 +1033,12 @@ class BigdataCitypriceBatch extends BaseBatch{ if(regionPriceMap){ for(def m : datas){ def regionAvgPrice = regionPriceMap.get(m.region) - def val = (m.avg_price-regionAvgPrice)*100/regionAvgPrice - distRegionRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val)) + if(regionAvgPrice){ + def val = (m.avg_price-regionAvgPrice)*100/regionAvgPrice + distRegionRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val)) + }else{ + println m.region+" 无区域均价" + } } } return distRegionRateMap; From 0831c011fca5662411f7e11f4877ad36a33d11e2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 7 Apr 2023 12:20:32 +0800 Subject: [PATCH 07/16] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E5=9F=8E=E5=B8=82?= =?UTF-8?q?=EF=BC=8C=E5=8C=BA=E5=9F=9F=E8=A1=8C=E6=83=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 726039ce9..054f27ec0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-12), "yyyyMMdd").toInteger(); - cityRun(citys, startDate, endDate, cityCodeDataMap) - regionRun(citys, startDate, endDate, cityCodeDataMap) +// cityRun(citys, startDate, endDate, cityCodeDataMap) +// regionRun(citys, startDate, endDate, cityCodeDataMap) districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap) } From 3dd084d77da8164c99b2f28dd8c974778d6e2eae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 7 Apr 2023 17:34:30 +0800 Subject: [PATCH 08/16] update --- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- .../ronghe/BigdataCitypriceBatch.groovy | 7 ++- .../ronghe/ExportDistrictRongHe.groovy | 17 +++++++- .../ronghe/ExportDistrictRongHeTest.groovy | 4 +- .../fangjia/script/gj/AddAvgpriceBatch.groovy | 43 ++++++++++++++++++- 5 files changed, 63 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 4035c2270..9515d1d1b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -440,7 +440,7 @@ class CcbCommonUtils { } // def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000) - println "原始:"+dbsour.count() + println "[${city}] 原始:"+dbsour.count() def rets = [] for(def d : dbsour){ if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ @@ -450,7 +450,7 @@ class CcbCommonUtils { rets << d } } - println "小区量:"+rets.size() + println "[${city}] 小区量:"+rets.size() return rets; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 054f27ec0..12f5ce551 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -57,13 +57,12 @@ class BigdataCitypriceBatch extends BaseBatch{ curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger(); def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger(); def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(curDate,-1), "yyyyMMdd").toInteger(); - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-12), "yyyyMMdd").toInteger(); // cityRun(citys, startDate, endDate, cityCodeDataMap) // regionRun(citys, startDate, endDate, cityCodeDataMap) - districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap) + districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } def cityRun(citys, startDate, endDate, cityCodeDataMap){ @@ -460,14 +459,14 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealYoy = '' if(allLastMonthMap){ def dealMarkupMap = allLastMonthMap.get(dist._id.toString()) - if(dealMarkupMap && dealMarkupMap.deal_count){ + if(dealMarkupMap && dealMarkupMap.deal_count && dealMarkupMap.deal_count>0){ def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count dealMarkup = DataHandleUtils.getDoubleRound(2, markup) } } if(allLastYearMap){ def dealYoyMap = allLastYearMap.get(dist._id.toString()) - if(dealYoyMap && dealYoyMap.deal_count){ + if(dealYoyMap && dealYoyMap.deal_count && dealYoyMap.deal_count>0){ def markup = (dealCount-dealYoyMap.deal_count)*100/dealYoyMap.deal_count dealYoy = DataHandleUtils.getDoubleRound(2, markup) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index eb5464e4d..e25da703d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -85,7 +85,7 @@ class ExportDistrictRongHe extends BaseBatch{ exportPgBigdataModelingBase(lastMonth) //2月8号 临时导出苏州一年的数据 - exportPgBigdataForsale("2023-01-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 + exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出 } SSHHelper jschCommand = new SSHHelper(); @@ -1669,7 +1669,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";// + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE,DEAL_COUNT,DEAL_COUNT_YOY,DEAL_COUNT_MOM,NETLIST_PRICE_YOY,NETLIST_PRICE_MOM,NETLIST_COUNT_YOY,NETLIST_COUNT_MOM,RENT_COUNT_YOY,RENT_COUNT_MOM,CITY_RANK,AREA_RANK,DISTRICT_PRICE_THAN,CITY_PRICE_THAN from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";// sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -1697,6 +1697,19 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.RDATERANGE data << r.DATATYPE data << r.SOURCECODE + data << r.DEAL_COUNT + data << r.DEAL_COUNT_YOY + data << r.DEAL_COUNT_MOM + data << r.NETLIST_PRICE_YOY + data << r.NETLIST_PRICE_MOM + data << r.NETLIST_COUNT_YOY + data << r.NETLIST_COUNT_MOM + data << r.RENT_COUNT_YOY + data << r.RENT_COUNT_MOM + data << r.CITY_RANK + data << r.AREA_RANK + data << r.DISTRICT_PRICE_THAN + data << r.CITY_PRICE_THAN allRows << data }); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index c274629b5..5b6c66379 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -26,7 +26,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ @Override void process(Object args) { citys = CcbCommonUtils.getRunningCitys(args) - def curDate = "20221210"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def curDate = DateUtils.tjaformatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) @@ -38,7 +38,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ exportBigdataCommlayoutpicFjw() - exportBigdataCommunityGJRH()// + exportBigdataCommunityGJRH()/ exportBigdataBuildingFJW()// exportBigdataCommfeeGj()// diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 7a531b0e7..7ed37c86c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -18,13 +18,13 @@ class AddAvgpriceBatch { public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static { - citys = CommonDatautils.findAllCity() + citys = ["徐州"]//CommonDatautils.findAllCity() println "加载城市完成:"+citys.size() } static main(args) { def task = new AddAvgpriceBatch(); - task.run(); + task.run1(); } def run(){ @@ -63,6 +63,45 @@ class AddAvgpriceBatch { + def run1(){ + + def totalCount = 0; + citys.each{city-> + if(city!="全国"){ + def districtKeysMap = CommonDatautils.findAllDistrictKeys(city, "district") + println "------------------------------------------------------------" + println "${city}小区量:"+districtKeysMap.size() + if(districtKeysMap){ + def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26")) + println "【2023-03-26】共计均价小区:"+lastWeekendStats.size() + //库中小区没有计算过均价的小区 + def districtKeys = districtKeysMap.keySet(); + def rets1 = findNoCaluteDistrict(districtKeys, lastWeekendStats); + println "【2023-03-26】共 "+rets1.size()+" 个小区无均价" + if(rets1){ + for(def disKey : rets1){ + def array = disKey.split("##"); + def estateType2 = districtKeysMap.get(disKey) + //发重算均价命令 + addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd')) + } + } + } + } + } + println "共补:"+totalCount + } + + + def addPrice(city, region, name, estateType2,startDate, endDate){ + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + for(def weekend : weekends){ + CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0) + } + } + + + def findHavePrice(lastAgoWeekendStats, rets){ def disKeys = []; for(def key : rets){ From dadd63c81929c207bcd4f78f28fb6c2088612032 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 7 Apr 2023 18:49:52 +0800 Subject: [PATCH 09/16] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCitypriceBatch.groovy | 14 +++- .../ronghe/UpdateCommPrice.groovy | 74 +++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/UpdateCommPrice.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 12f5ce551..952218baa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject import groovy.sql.Sql @@ -459,14 +460,19 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealYoy = '' if(allLastMonthMap){ def dealMarkupMap = allLastMonthMap.get(dist._id.toString()) - if(dealMarkupMap && dealMarkupMap.deal_count && dealMarkupMap.deal_count>0){ - def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count - dealMarkup = DataHandleUtils.getDoubleRound(2, markup) + try{ + if(dealCount && dealMarkupMap && dealMarkupMap.deal_count && dealMarkupMap.deal_count>0){ + def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count + dealMarkup = DataHandleUtils.getDoubleRound(2, markup) + } + }catch(Exception e){ + e.printStackTrace() } + } if(allLastYearMap){ def dealYoyMap = allLastYearMap.get(dist._id.toString()) - if(dealYoyMap && dealYoyMap.deal_count && dealYoyMap.deal_count>0){ + if(dealCount && dealYoyMap && dealYoyMap.deal_count && dealYoyMap.deal_count>0){ def markup = (dealCount-dealYoyMap.deal_count)*100/dealYoyMap.deal_count dealYoy = DataHandleUtils.getDoubleRound(2, markup) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/UpdateCommPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/UpdateCommPrice.groovy new file mode 100644 index 000000000..a69e66d8f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/UpdateCommPrice.groovy @@ -0,0 +1,74 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +class UpdateCommPrice { + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + + def oracleSqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + public static def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + static main(args) { + def task = new UpdateCommPrice() + task.run(args) + } + + def run(Object args){ + def citys = CcbCommonUtils.citys_all + def curDate = new Date() + if(args){ + def dateStr = args[0] + curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01 + } + def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger(); + def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger(); + def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) + + process(citys, dealStartDate, startDate, curRunDate) + + } + + def process(citys, dealStartDate, startDate, curRunDate){ + def rets = [] + def allDistricts = [] + def allDealCountMap = [:] + for(def city : citys) { + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias"]); + allDistricts.addAll(districts) + + def cityCode = cityCodeMap.get(city) + def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate) + allDealCountMap = allDealCountMap + dealCountMap + } + + for(def i=0;i + rMap.put(r.gujia_comm_id, r.count) + }) + return rMap; + } +} From f33f728a8802545e840a7b7d553ac5cf9048eb90 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 8 Apr 2023 14:46:06 +0800 Subject: [PATCH 10/16] =?UTF-8?q?=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 952218baa..fe7480709 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -60,9 +60,9 @@ class BigdataCitypriceBatch extends BaseBatch{ def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(curDate,-1), "yyyyMMdd").toInteger(); def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) - def lastYearDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-12), "yyyyMMdd").toInteger(); -// cityRun(citys, startDate, endDate, cityCodeDataMap) -// regionRun(citys, startDate, endDate, cityCodeDataMap) + def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); +// cityRun(citys, startDate, endDate, cityCodeMap) +// regionRun(citys, startDate, endDate, cityCodeMap) districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } @@ -271,6 +271,7 @@ class BigdataCitypriceBatch extends BaseBatch{ def rets = new CopyOnWriteArrayList() for(def i=0;i Date: Sat, 8 Apr 2023 22:54:15 +0800 Subject: [PATCH 11/16] update bug --- .../ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index fe7480709..9ed8300cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -502,7 +502,7 @@ class BigdataCitypriceBatch extends BaseBatch{ def dataMap = allLastYearMap.get(dist._id.toString()) if(dataMap && dataMap.saleprice){ yoy = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice - yoy = DataHandleUtils.getDoubleRound(2, markup) + yoy = DataHandleUtils.getDoubleRound(2, yoy) } } row << yoy @@ -527,7 +527,7 @@ class BigdataCitypriceBatch extends BaseBatch{ def dataMap = allLastYearMap.get(dist._id.toString()) if(dataMap && dataMap.salecount){ yoy = (sell_count - dataMap.salecount)*100/dataMap.salecount - yoy = DataHandleUtils.getDoubleRound(2, markup) + yoy = DataHandleUtils.getDoubleRound(2, yoy) } } row << yoy @@ -552,7 +552,7 @@ class BigdataCitypriceBatch extends BaseBatch{ def dataMap = allLastYearMap.get(dist._id.toString()) if(dataMap && dataMap.salecount && dataMap.rentcount>0){ yoy = (rent_count - dataMap.rentcount)*100/dataMap.rentcount - yoy = DataHandleUtils.getDoubleRound(2, markup) + yoy = DataHandleUtils.getDoubleRound(2, yoy) } } row << yoy From bc77fcddead8bfd70d08172ce852e7ddbc440a28 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 9 Apr 2023 10:40:37 +0800 Subject: [PATCH 12/16] update --- .../ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 5b6c66379..8417fd237 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -26,7 +26,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ @Override void process(Object args) { citys = CcbCommonUtils.getRunningCitys(args) - def curDate = DateUtils.tjaformatDateByPattern(new Date()+1, "yyyyMMdd") + def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) @@ -854,7 +854,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate} + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID>=202210";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID From c18126997ad6b1350b3583af9bb6ee421bea8031 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 9 Apr 2023 12:03:18 +0800 Subject: [PATCH 13/16] test --- .../ronghe/ExportDistrictRongHeTest.groovy | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 8417fd237..c93aeae8d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -29,17 +29,17 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ - exportEvaluateRate() - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - - - exportBigdataCommunityGJRH()/ - exportBigdataBuildingFJW()// +// exportEvaluateRate() +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// +// +// exportBigdataCommunityGJRH()/ +// exportBigdataBuildingFJW()// exportBigdataCommfeeGj()// exportBigdataCommbuildstruGJ()// From 66ab3f7840879829b9b3875e84aeb3d4702c3b8a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 9 Apr 2023 12:18:11 +0800 Subject: [PATCH 14/16] =?UTF-8?q?=E8=A1=8C=E6=83=85=E8=A1=A8=E6=94=B9?= =?UTF-8?q?=E8=A1=A8=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index c93aeae8d..700037491 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -760,7 +760,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ for(def city : citys) { def cityCode = cityCodeMap.get(city) // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// - def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}'";// sqlInstance.eachRow(sql,{r-> def data = [] @@ -807,7 +807,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate} + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}'";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -854,7 +854,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID>=202210";// and YMID=${queryDate} + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID>=202210";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID From 42825c90c6efad4676d0ea24f546c1980e5c20a9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 9 Apr 2023 12:50:27 +0800 Subject: [PATCH 15/16] =?UTF-8?q?=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHeTest.groovy | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 700037491..9a5ae3090 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -29,7 +29,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ @@ -40,22 +40,22 @@ class ExportDistrictRongHeTest extends BaseBatch{ // // exportBigdataCommunityGJRH()/ // exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// - - - //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 - def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// +// +// +// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 +// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) exportInOk() From ca9852a0825857816d7386fe2681fbba74169343 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 9 Apr 2023 13:19:02 +0800 Subject: [PATCH 16/16] =?UTF-8?q?=E5=8F=A6=E4=B8=80=E4=BB=BD=E6=88=90?= =?UTF-8?q?=E4=BA=A4=E6=8D=A2=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHeTest.groovy | 132 +++++++++++++++++- 1 file changed, 131 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 9a5ae3090..0f2519b79 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -5,6 +5,7 @@ import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils @@ -14,6 +15,9 @@ import groovy.sql.Sql @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHeTest extends BaseBatch{ + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { @@ -29,7 +33,8 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ @@ -528,6 +533,50 @@ class ExportDistrictRongHeTest extends BaseBatch{ println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" } + + + def exportPgBigdataDealFJW(curDate, dealdate, new_dists){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + + + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(curDate.toString().endsWith("10")){ + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealdate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";// and txn_dt>='${dealdate}' + allRows.addAll(dealData(sql)) + } + } + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + } + + def createDeal(sql){ def allRows = [] sqlInstance.eachRow(sql,{r-> @@ -607,6 +656,87 @@ class ExportDistrictRongHeTest extends BaseBatch{ return allRows; } + + def dealData(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_bds_mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_bds_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujiabdstot_floor_num + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_bds_trdtckt_prc + data << r.gujia_bds_mdl_tot_prc + data << r.gujia_bds_dcl_tot_prc + data << r.gujiabds_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_bds_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiabdslndfgrngvarea + data << r.gujiabdslutbl_gv_area + data << r.gujiabdssuntblgv_area + data << r.gujiabdsbayindowgarea + data << r.gujiabdstowerfrgvarea + data << r.gujia_bds_grg_gv_area + data << r.gujiabdsundgdrmgvarea + data << r.gujiabds_othr_gv_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co +// data << r.gujia_bds_mkdam + data << r.gujia_bds_data_tm_val + allRows << data + }); + return allRows + } + def exportAllBigdataDealFJW(){ def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE