常州脚本修改

This commit is contained in:
2023-04-04 11:18:26 +08:00
parent 42553cf738
commit f5d39e53c4
7 changed files with 654 additions and 36 deletions
@@ -33,7 +33,7 @@ class FjwPricePkCcbPrice {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
// task.addFjwPrice()
task.addFjwPrice()
// task.addCcbPrice()
// task.addCcbPriceFromGuided()
@@ -44,7 +44,7 @@ class FjwPricePkCcbPrice {
// task.addCcbPrice2()
// task.addCcbPrice3()
task.standRegion()
// task.standRegion()
}
def addCcbPrice2(){
@@ -202,13 +202,13 @@ class FjwPricePkCcbPrice {
}
def addFjwPrice(){
ExcelUtils.modify("D:\\数据\\建行\\测试\\人工查价(7455)_0824_0913.xlsx",{d->
if(!d['M']){
def dist = CommonDatautils.findDistrictById(d['D'])
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\3月国信达差价_755条.xlsx",{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2022-09-11","yyyy-MM-dd"))
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-03-26","yyyy-MM-dd"))
if(dbo){
d['M'] = dbo.avg_price
d['K'] = dbo.avg_price
}
}
}
@@ -0,0 +1,40 @@
package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
def rMap = [:]
def index = 0
OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
index++
def day = doc.c_date.format("yyyy-MM-dd")
def count = rMap.get(day)
if (!count) {
rMap.put(day, 1)
} else {
rMap.put(day, count + 1)
}
if(index%10000==0){
println index+" "+new Date().toString()
}
}
rMap.each {k,val->
println k+" "+val
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.linxsh.util.AddressUtil
@@ -11,26 +12,67 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AnjukeDistrictMatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new AnjukeDistrictMatch();
// task.run();
task.matchCommunity()
// task.runCsv();
// task.updateCredit()
// task.matchCommunity()//查询es小区数据
// task.run1()
// task.updateField()
// task.updateGreenRateField()//根据第三方数据填补库中字段
// task.updteGreenRateFieldByCsv()
// task.fenQiRun()
// task.findAliasDatas("淮安","水沐未来印象花园")
// task.findAliasDatas("淮安","水沐未来印象花园")//模糊查找Es别名数据
// task.repeatDistByAlias()//根据第三方网站别名查找重复数据
task.matchCommunityByCsv()
}
def updateCredit(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv")
def results = []
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[24] && row[24].contains("接口")){
def inputName = row[2]
def matchName = row[27]
if(inputName.toString().endsWith("小区") || inputName.toString().endsWith("院")){
inputName = inputName.replace("小区","").replace("院","")
}
if(matchName.toString().endsWith("小区") || matchName.toString().endsWith("院")){
matchName = matchName.replace("小区","").replace("院","")
}
if(inputName == matchName && row[1]==row[26]){
row[30] = 1
}
}
results << row
}
batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-1.csv",null,"GBK")
batchCreateFile(datas, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-2.csv",null,"GBK")
}
@@ -96,7 +138,7 @@ class AnjukeDistrictMatch {
}
def updateField(){
def updateGreenRateField(){
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['Q'] && !row['Q'].contains("接口")){
@@ -155,18 +197,242 @@ class AnjukeDistrictMatch {
})
}
def matchCommunityByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-1.csv")
def map = [:]
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
map.put(row[0]+"#"+row[1]+"#"+row[2], row[24]+"#"+row[25]+"#"+row[26]+"#"+row[27]+"#"+row[28])
}
def datas1 = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站建筑面积.csv")
def lastRets = []
for(def i=0;i<datas1.size();i++) {
final def data = datas1[i]
def row = data as List
def resultStr = map.get(row[0]+"#"+row[1]+"#"+row[2])
if(resultStr){
def arrs = resultStr.split("#")
if(arrs){
row << arrs[0]
row << arrs[1]
if(arrs.size()>2){
row << arrs[2]
}
if(arrs.size()>3){
row << arrs[3]
}
if(arrs.size()>4){
row << arrs[4]
}
}
}
lastRets << row
}
batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站建筑面积(匹配).csv",null,"GBK")
}
def updteGreenRateFieldByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\测试.csv")
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[24] && (row[24]=="名称匹配"||row[24]=="映射别名匹配" || (row[24].contains("接口") && row[28] && row[28].toString().isNumber() && row[28].toString().toDouble()>=0.9))){
def dist = CommonDatautils.findSeaweedById(row[25], ["virescence_percent"])
if(dist && !dist.virescence_percent){
def val = row[12]
if(!val){
val = row[13]
}
if(!val){
val = row[14]
}
if(!val){
val = row[11]
}
if(!val){
val = row[21]
}
if(!val){
val = row[22]
}
if(!val){
val = row[6]
}
if(!val){
val = row[10]
}
if(!val){
val = row[9]
}
if(!val){
val = row[5]
}
if(!val){
val = row[16]
}
if(!val){
val = row[15]
}
if(!val){
val = row[23]
}
if(!val){
val = row[19]
}
if(!val){
val = row[17]
}
if(!val){
val = row[8]
}
if(!val){
val = row[18]
}
if(!val){
val = row[20]
}
if(!val){
val = row[3]
}
if(!val){
val = row[4]
}
if(!val){
val = row[7]
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("virescence_percent", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
println row[25] +" "+val.toString()
}
}
}
}
}
// def run1(){
// def regionPointsMap = findRegionRange("重庆")
// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
// }
def runCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")//第三方网站绿化率 (3).csv
def results = batchMatch(datas)
batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv",null,"GBK")
}
def batchMatch(datas){
def results = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def data = datas[i]
def row = data as List
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if (row[0] && row[1]) {
def city = row[0]
def region = row[1]
def name = row[2]
region = RegionData.formatRegion(city, region)
if(region){
if (name && name.contains("·")) {
name = name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(city, region, name)
if (!dist) {
dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name, "district")
flag = "映射别名匹配"
}
if (!dist) {
def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if (rets && rets.size() == 1) {
dist = rets[0]
}
}
if (dist) {
row << flag
row << dist._id.toString()
row << dist.region
row << dist.name
results << row
} else {
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null)
if (jsonObject != null && jsonObject.precise != -1) {
row << "匹配接口"
row << jsonObject.get("id")
row << jsonObject.get("district")
row << jsonObject.get("name")
row << jsonObject.get("credit")
results << row
}
}
}
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return results;
}
def run(){
def filePath = "D:\\数据\\小区\\押品发现新增小区确认.xlsx"
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\全国绿化率.xlsx"
def index = 0
ExcelUtils.multiThreadingModify(filePath,{row->
index++
if(row['A'] && row['B']){
def city = row['A']
def name = row['B']
def region = null
def name = row['C']
def region = row['B']
if(name && name.contains("·")){
name = name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(city, region, name)
@@ -203,26 +469,26 @@ class AnjukeDistrictMatch {
}else{
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = "匹配接口"
row['G'] = jsonObject.get("id")
row['D'] = jsonObject.get("district")
row['C'] = jsonObject.get("name")
row['Y'] = "匹配接口"
row['Z'] = jsonObject.get("id")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
row['AC'] = jsonObject.get("credit")
}
}
if(row['G']){
def dbo = CommonDatautils.findDistrictById(row['G'])
if(dbo){
row['H'] = dbo.estate_type2
row['I'] = dbo.name_cat
def count = findBigdataCommunity(row['G'])
if(count){
row['E'] = "是"
}
}
}
println index +" "+row['Y']
// if(row['G']){
// def dbo = CommonDatautils.findDistrictById(row['G'])
// if(dbo){
// row['H'] = dbo.estate_type2
// row['I'] = dbo.name_cat
//
// def count = findBigdataCommunity(row['G'])
// if(count){
// row['E'] = "是"
// }
// }
// }
}
})
}
@@ -286,6 +552,57 @@ class AnjukeDistrictMatch {
return rets;
}
def repeatDistByAlias(){
def rets = findAliasDist()
// def datas = []
// for(def r : rets){
// datas << [r.city, r.region, r.name, r.alias]
// }
// ExcelUtils.write("D:\\数据\\小区\\第三方有别名的小区.xlsx", datas, "test")
def districts = CommonDatautils.findAllDistrictByCity("常州", "district", ["city","region","name"])
def distKeys = []
for(def d : districts){
distKeys << d.region+"#"+d.name
}
def results = []
for(def t : rets){
def names = []
names << t.region + "#" +t.name
def arrs = t.alias.split(",|,")
for(def a : arrs){
names << t.region + "#" + a
if(a.contains("(")){
names << t.region + "#" + a.replace("(","").replace(")", "")
}
}
def set = new HashSet()
for(def n : names){
if(distKeys.contains(n)){
set << n
}
}
if(set.size()>1){
results << [t.city, t.region, t.name, set.join(" "),t.url]
}
}
ExcelUtils.write("D:\\数据\\小区\\第三方别名验证重复小区.xlsx", results, "test")
}
def findAliasDist(){
def rets = []
def sql = "select * from community where city='常州'"
ElasticUtils.executeSqlQuery(sql, { d->
if(d.alias){
rets << ["city":d.city, "region":d.region, "name":d.name, "source":d.source, "alias":d.alias, "url":d.url]
}
})
return rets;
}
def findAliasDatas(city, name){
def must = []
def term1 = ["match":["city":city]]
@@ -309,4 +626,49 @@ class AnjukeDistrictMatch {
}
return rets;
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split(",",-1)
rows << row
}
return rows;
}
def batchCreateFile(ret, fileName, headers, charset){
OutputStream out = new FileOutputStream(fileName);
def sb = new StringBuffer();
// sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator"));
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
index++
sb.append(StringUtils.join(row, ","))
sb.append(System.getProperty("line.separator"));
println index
if(index%10000==0){
println index+"开始写入CSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"结束写入CSV "+new Date().toLocaleString()
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -0,0 +1,131 @@
package com.fangjia.script.gj.poc
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class AddPriceTest {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
static main(args) {
def task = new AddPriceTest();
task.run();
}
def run(){
def filePath = "D:\\数据\\招商\\招商.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
addThirdPrice(filePath)
}
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['M']){
def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['M'], cityId)
if(map){
d['AK'] = map.avg_price
d['AL'] = map.calc_type
d['AM'] = map.credit_value
}
}
})
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['M']) {
def dist = CommonDatautils.findDistrictById(d['M'])
if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['AN'] = thirdPrice
}
}
})
}
def findThirdPartyPrice(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null
for(def db : dbcursor){
if(!price && db.lianjia_esf_price && db.lianjia_esf_price>2000){
price = db.lianjia_esf_price
break;
}
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
price = db.beike_esf_price
break
}
}
return price
}
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['M']) {
def dist = CommonDatautils.findDistrictById(d['M'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
d['AO'] = gxdPrice
}
}
})
}
def findGXDPrice(def city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("source", "GXD")
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
def price = null
for(def d : dbcursor){
price = d.price
}
return price
}
}
@@ -0,0 +1,84 @@
package com.fangjia.script.gj.poc
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class RentPrice {
static main(args) {
def task = new RentPrice();
task.run("D:\\数据\\招商\\楼盘POC测试样例地址.xlsx");
}
def run(filepath){
def index = 0
ExcelUtils.multiThreadingModify(filepath, { d ->
index++
if(d['M']){
def dist = CommonDatautils.findDistrictById(d['M'])
if(dist){
d['AZ'] = findHouses(dist.city, dist.region, "rent", 20221201,20230330, dist.name, dist.alias)
}
println index + " "+ d['AZ']
}
})
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
// BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
// inBuilder.should(QueryBuilders.matchQuery("root", "lianjia.com"));
// inBuilder.should(QueryBuilders.matchQuery("root", "ke.com"));
// queryBuilder.must(inBuilder)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def prices = []
println city + " "+name+" "+houses.size()
for(def d : houses){
prices << d.avg_price
}
if(prices && prices.size()>=3){
return (int)(prices.sum()/prices.size())
}
return null
}
}
@@ -32,8 +32,9 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class CZDataBatch extends BaseBatch{
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
//final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls");
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303")