Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+42
-42
@@ -67,50 +67,50 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
println "curDate:"+curDate
|
||||
// for(def city : citys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
// }
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
saveToPgSql(rows)
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
// saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
|
||||
+2
-2
@@ -375,12 +375,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202301
|
||||
def month = 202302
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202302)
|
||||
def rets = findPeGteThirdParty2(city, 202303)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
|
||||
+9
-4
@@ -217,10 +217,15 @@ class FjwPricePkCcbPrice {
|
||||
}
|
||||
|
||||
def standRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入3月20日.xlsx",{d->
|
||||
if(!d['J']){
|
||||
def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
d['J'] = newRegion
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\高德写字楼.xlsx",{d->
|
||||
if(d['A']){
|
||||
def city = RegionData.formatCity(d['A'])
|
||||
if(city){
|
||||
def newRegion = RegionData.formatRegion(city, d['B'])
|
||||
d['J'] = city
|
||||
d['K'] = newRegion
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+40
-40
@@ -43,49 +43,49 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
// def rows = []
|
||||
//
|
||||
// CcbCommonUtils.deleteGuidedPrice(curDate)
|
||||
// for (def city : citys) {
|
||||
// def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
//
|
||||
// //添加自己库中的人工指导价
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-27")
|
||||
//// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
// if(prices){
|
||||
// def distMap = getDistrictMap(city)
|
||||
// def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
// rows.addAll(rets)
|
||||
// println city + " " + rets.size()
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
CcbCommonUtils.deleteGuidedPrice(curDate)
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-02-27", "2023-03-21")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
|
||||
//添加搜房的参考价
|
||||
// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
// def datas = findFangDatas(districts)
|
||||
// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
|
||||
// rows.addAll(fangRets)
|
||||
|
||||
//添加保亭文昌的指导价
|
||||
// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
|
||||
// def cityDataMap = [:]
|
||||
// for(def d : cityDatas){
|
||||
// cityDataMap.put(d['A'], d['E'])
|
||||
// }
|
||||
//
|
||||
// //添加搜房的参考价
|
||||
//// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
//// def datas = findFangDatas(districts)
|
||||
//// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
|
||||
//// rows.addAll(fangRets)
|
||||
//
|
||||
// //添加保亭文昌的指导价
|
||||
//// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
|
||||
//// def cityDataMap = [:]
|
||||
//// for(def d : cityDatas){
|
||||
//// cityDataMap.put(d['A'], d['E'])
|
||||
//// }
|
||||
//// def personDatas = findPersonDatas(districts, cityDataMap);
|
||||
//// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
|
||||
//// rows.addAll(personRets)
|
||||
//
|
||||
// }
|
||||
//
|
||||
// def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
//
|
||||
// saveToPgSql(rows)
|
||||
// def personDatas = findPersonDatas(districts, cityDataMap);
|
||||
// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
|
||||
// rows.addAll(personRets)
|
||||
|
||||
}
|
||||
|
||||
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false);
|
||||
|
||||
saveToPgSql(rows)
|
||||
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
}
|
||||
|
||||
|
||||
+5
-5
@@ -92,13 +92,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","11月总量","12月总量","23年1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","12月总量","23年1月总量","23年2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-11-01"
|
||||
def date = "2022-12-01"
|
||||
def endDate = "2023-01-01"
|
||||
def newDate = "2023-02-01"
|
||||
def startDate = "2022-12-01"
|
||||
def date = "2023-01-01"
|
||||
def endDate = "2023-02-01"
|
||||
def newDate = "2023-03-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+1
-1
@@ -390,7 +390,7 @@ class ExportPgsqlEvaluationResult {
|
||||
//
|
||||
// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
// }
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload(null, fileName, curDate)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -886,7 +886,7 @@ class CommonDatautils {
|
||||
keys.put("mapping_alias", 1)
|
||||
keys.put("sid", 1)
|
||||
keys.put("status", 1)
|
||||
|
||||
keys.put("name_cat", 1)
|
||||
try{
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
class AddDistrictTemp {
|
||||
|
||||
static main(args) {
|
||||
def task = new AddDistrictTemp();
|
||||
task.run();
|
||||
}
|
||||
def run() {
|
||||
def datas = ExcelUtils.read("D:\\数据\\小区\\新增小区.xlsx", 0, 1)
|
||||
def index = 0
|
||||
for (def d : datas) {
|
||||
index ++
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("name", d['C'])
|
||||
q.put("cat","district")
|
||||
q.put("status", 0)
|
||||
def obj = DbUtils.seaweed.findOne(q)
|
||||
if(!obj){
|
||||
def dbo = new BasicDBObject()
|
||||
dbo.city = d['A'];
|
||||
dbo.region = d['B'];
|
||||
dbo.name = d['C']
|
||||
dbo.address = d['F']
|
||||
dbo.estate_type2 = "普通住宅"
|
||||
dbo.status = 0;
|
||||
dbo.cat ="district"
|
||||
dbo.name_cat = 0
|
||||
dbo.c_date = new Date()
|
||||
dbo.m_date = new Date()
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
dbo.mark_map = markMap
|
||||
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
Long lng_new = d['G'].toString().toDouble() *10000000000L
|
||||
Long lat_new = d['H'].toString().toDouble() *10000000000L
|
||||
dbo.put("lng", lng_new)
|
||||
dbo.put("lat", lat_new)
|
||||
if(lng2){
|
||||
Long lng2_new = lng2 *10000000000L
|
||||
Long lat2_new = lat2 *10000000000L
|
||||
dbo.put("lng2", lng2_new)
|
||||
dbo.put("lat2", lat2_new)
|
||||
}
|
||||
def alias = []
|
||||
if(d['D'] && d['D']!=d['C'] && !alias.contains(d['D'])){
|
||||
alias << d['D']
|
||||
}
|
||||
|
||||
if(d['F'] && d['F']!=d['C'] && !alias.contains(d['F'])){
|
||||
alias << d['F']
|
||||
}
|
||||
if(alias){
|
||||
dbo.put("alias", alias)
|
||||
}
|
||||
|
||||
println "[${index}]"+dbo
|
||||
DbUtils.seaweed.save(dbo);
|
||||
}else{
|
||||
println d['C']+" 已存在"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -539,4 +539,20 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
|
||||
static def searchDistrictRank(def city, def region, def name){
|
||||
def url = "http://localhost:8080/fangjia-open/property/rank?city="+city+"&district="+region+"&name="+name+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def dto = []
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+123
-19
@@ -1,23 +1,40 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class AnjukeDistrictMatch {
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeDistrictMatch();
|
||||
// task.run();
|
||||
task.matchCommunity()
|
||||
// task.run1()
|
||||
// task.updateField()
|
||||
task.fenQiRun()
|
||||
// task.fenQiRun()
|
||||
|
||||
// task.findAliasDatas("淮安","水沐未来印象花园")
|
||||
}
|
||||
|
||||
|
||||
|
||||
def fenQiRun(){
|
||||
def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
|
||||
for(def c : citys){
|
||||
@@ -144,20 +161,16 @@ class AnjukeDistrictMatch {
|
||||
// }
|
||||
def run(){
|
||||
|
||||
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
|
||||
def filePath = "D:\\数据\\小区\\押品发现新增小区确认.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath,{row->
|
||||
if(row['A'] && !row['Q']){
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = row['D']
|
||||
if(row['A'] && row['B']){
|
||||
def city = row['A']
|
||||
def name = row['B']
|
||||
def region = null
|
||||
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(city, region, name)
|
||||
if(!row['D'] && name.contains("(")){
|
||||
name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||
flag = "去括号匹配"
|
||||
dist = matchDist(city, region, name)
|
||||
}
|
||||
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
|
||||
flag = "映射别名匹配"
|
||||
@@ -183,20 +196,59 @@ class AnjukeDistrictMatch {
|
||||
// }
|
||||
|
||||
if(dist){
|
||||
row['Q'] = flag
|
||||
row['R'] = dist._id.toString()
|
||||
row['S'] = dist.region
|
||||
row['T'] = dist.name
|
||||
row['F'] = flag
|
||||
row['G'] = dist._id.toString()
|
||||
row['D'] = dist.region
|
||||
row['C'] = dist.name
|
||||
}else{
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['Q'] = "匹配接口"
|
||||
row['R'] = jsonObject.get("id")
|
||||
row['S'] = jsonObject.get("district")
|
||||
row['T'] = jsonObject.get("name")
|
||||
row['F'] = "匹配接口"
|
||||
row['G'] = jsonObject.get("id")
|
||||
row['D'] = jsonObject.get("district")
|
||||
row['C'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
|
||||
if(row['G']){
|
||||
def dbo = CommonDatautils.findDistrictById(row['G'])
|
||||
if(dbo){
|
||||
row['H'] = dbo.estate_type2
|
||||
row['I'] = dbo.name_cat
|
||||
|
||||
def count = findBigdataCommunity(row['G'])
|
||||
if(count){
|
||||
row['E'] = "是"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchCommunity(){
|
||||
def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)3.24_2.xlsx"
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if (row['A'] && row['B'] && row['I']) {
|
||||
def rets = findAllEsData(row['A'], row['B'])
|
||||
if(rets){
|
||||
def dbo = rets[0]
|
||||
row['J'] = dbo.city
|
||||
row['K'] = dbo.region
|
||||
row['L'] = dbo.name
|
||||
row['M'] = dbo.alias
|
||||
row['N'] = dbo.source
|
||||
row['O'] = dbo.score
|
||||
if(dbo.address){
|
||||
row['P'] = AddressUtil.formatAddress(dbo.city, dbo.address)
|
||||
}
|
||||
if(dbo.location){
|
||||
row['Q'] = dbo.location.lon
|
||||
row['R'] = dbo.location.lat
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -205,4 +257,56 @@ class AnjukeDistrictMatch {
|
||||
def dist = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
return dist;
|
||||
}
|
||||
|
||||
|
||||
def findBigdataCommunity(comm_id){
|
||||
def sql = "select count(1) as count from base.t0831_bigdata_community_gj_h where gujia_comm_id='${comm_id}' ";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
def findAllEsData(city,name){
|
||||
def rets = []
|
||||
def sql = "select * from community where city='${city}' and name='${name}'"
|
||||
rets = findDatas(sql)
|
||||
if(!rets){
|
||||
rets = findAliasDatas(city,name)
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findDatas(sql){
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << ["city":d.city, "region":d.region, "name":d.name, "source":d.source, "alias":d.alias, "address":d.address, "location":d.location]
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAliasDatas(city, name){
|
||||
def must = []
|
||||
def term1 = ["match":["city":city]]
|
||||
must << term1
|
||||
|
||||
def should = []
|
||||
should << ["match":["alias":name]]
|
||||
def params = ["query":["bool":["must":must, "should":should]]]
|
||||
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/community/_doc/_search");
|
||||
def rets = [];
|
||||
if(s) {
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
hits = hits.hits
|
||||
for(def h : hits){
|
||||
def d = h._source
|
||||
rets << ["city":d.city, "region":d.region, "name":d.name, "source":d.source, "alias":d.alias, "score":h._score, "address":d.address, "location":d.location]
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class priceRank {
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new priceRank()
|
||||
task.run()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("上海","district", ["city","region","name"])
|
||||
//def dists = CommonDatautils.findDistrictByCityAndRegions("上海","district", ["浦东"], ["city","region","name"])
|
||||
def rets = createCityDatas(dists)
|
||||
for(def r : rets){
|
||||
println r
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\浦东小区排名.xlsx", rets, "测试")
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createCityDatas(def dists){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = []
|
||||
def priceDbo = CommonDatautils.queryDistrictPrice(d.city, d.region, d.name, DateUtils.parseFromTightByPattern('2023-03-19','yyyy-MM-dd'))
|
||||
def rankDbo = com.fangjia.script.gj.OpenApiUtils.searchDistrictRank(d.city, d.region, d.name)//findDistrictRank(d.city, d.region, d.name)
|
||||
ret << d.city
|
||||
ret << d.region
|
||||
ret << d.name
|
||||
if(priceDbo){
|
||||
ret << priceDbo.avg_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(rankDbo){
|
||||
ret << rankDbo.relativeRank
|
||||
ret << rankDbo.regionRelativeRank
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
rets << ret
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findDistrictRank(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
def dbo = DISTRICT_ANALY_DATA_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+2
-2
@@ -474,11 +474,11 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["福州"]
|
||||
def citys= ["苏州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") ){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,3 +41,12 @@ while (startDate.before(endDate)) {
|
||||
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
//def list = []
|
||||
//OPEN_URI_LOG.find(and(eq("user_id", "1007777"), eq("request_uri", "/property/avgPrice"))).each { Document doc ->
|
||||
// def result = doc.res_map.result
|
||||
// def item = [ "泉果", doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
// list << item
|
||||
//}
|
||||
//ExcelUtils.write("D:\\泉果接口查价.xlsx", list, "查询日志")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user