Merge remote-tracking branch 'origin/script_gj' into script_gj

This commit is contained in:
zhaojixiang@fangjia.com
2023-07-06 18:59:47 +08:00
29 changed files with 1452 additions and 166 deletions
@@ -33,7 +33,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
@@ -61,56 +61,58 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
// def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
def allCitys = ["福州"] //CommonDatautils.findAllCity()
for(def city : allCitys){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
}
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
saveToPgSql(rows)
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
//
// saveToPgSql(rows)
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -649,16 +651,21 @@ class BigdataDealBatch extends BaseBatch{
def final AtomicInteger count_ = new AtomicInteger(0)
for(def i=0;i<trades.size();i++){
def t = trades[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
final def t = trades[i]
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
count_.addAndGet(1)
try{
if(!t.match_id){
def region = RegionData.formatRegion(t.city, t.region)
def rMap = null
if(region) {
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
if(!rMap && t.address){
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
@@ -668,7 +675,12 @@ class BigdataDealBatch extends BaseBatch{
}
if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else {
if (rMap) {
println "区域不同"
}
}
}
println count_.get()+" "+t+" "+ rMap
}
@@ -678,9 +690,9 @@ class BigdataDealBatch extends BaseBatch{
latch.countDown()
}
}
}
);
// }
// }
// );
}
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
@@ -617,14 +617,14 @@ class CcbCommonUtils {
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate))
q.put("match_id", new BasicDBObject('$exists', false))
// q.put("district_name","国贸天琴湾")
// q.put("district_name","中海寰宇天下")
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
println city +" "+total
for(def d : dbcursor){
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id]
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id, "house_use":d.house_use,"lift_household":d.lift_household,"lift":d.lift]
}
return lists
}
@@ -28,11 +28,12 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()//删除第三方
task.deleteThirdPartyData()//删除第三方
// task.delDealByPe()
// task.delDealData()
task.exportThirdParty()
// task.exportThirdParty()
// task.statPpe()
}
def delDealByPe(){
@@ -379,7 +380,7 @@ class DealPriceDataProcess {
def results = []
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
for(def city : citys) {
def rets = findPeGteThirdParty3(city, 202305)
def rets = findPeGteThirdParty3(city, 202306)
for(def map : rets){
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
}
@@ -391,12 +392,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202304
def month = 202305
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202305)
def rets = findPeGteThirdParty2(city, 202306)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -490,5 +491,43 @@ class DealPriceDataProcess {
return count
}
def statPpe(){
def path = "C:\\Users\\gaojuan\\Desktop\\估值数据统计.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
def deal_pes = []
def third_pes = []
for(def d : dates){
def map = findPeGteCity(row['B'], d)
if(map.deal_pe){
deal_pes << map.deal_pe
}
if(map.third_pe){
third_pes << map.third_pe
}
}
if(deal_pes){
row['F'] = deal_pes.sum()/deal_pes.size()
}
if(third_pes){
row['G'] = third_pes.sum()/third_pes.size()
}
println row['B'] + " "+row['F']+" "+row['G']
})
}
def findPeGteCity(cityName, yearmonth){
def sql = "SELECT gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}"
def map = [:]
pgsqlInstance.eachRow(sql,{r->
map.deal_pe = r.gujiamdltst_ppe15_pct
map.third_pe = r.gujiatrdpttstppe15pct
});
return map
}
}
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//
// //更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -17,4 +17,14 @@ class PgsqlCommUtils {
});
return set
}
public static def findAllDistrictCount(def cityid){
def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
}
@@ -45,8 +45,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
//def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
// "postgres", "postgres", "org.postgresql.Driver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
//def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -1289,28 +1289,28 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
def region = RegionData.formatRegion(d['B'], d['C'])
if(region){
def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
if(dist){
d['G'] = dist.region
d['H'] = dist._id.toString()
}
}
// if(!d['F']){
// def dist = CommonDatautils.findDistrictById(d['A'])
////
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
// if(dist){
//// d['H'] = dist.region
//// d['G'] = dist.name
// d['F'] = dist.status
// d['G'] = dist.region
// d['H'] = dist._id.toString()
// }
// }
}
},0)
//// if(!d['F']){
//// def dist = CommonDatautils.findDistrictById(d['A'])
//////
//// if(dist){
////// d['H'] = dist.region
////// d['G'] = dist.name
//// d['F'] = dist.status
//// }
//// }
//
// }
//},0)
@@ -1364,6 +1364,53 @@ def findGuidedPrice(sqlInstance, id){
return count
}
def findPgDealPrice(c, pgsqlInstance, startDate){
def sql = "select count(a.gujia_comm_id) from (SELECT distinct gujia_comm_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${c}' and txn_dt>='${startDate}') a"
def count = 0
pgsqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
citys = CcbCommonUtils.citys_all
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
def rows = []
def headers = []
headers << "城市"
headers.addAll(dates)
rows << headers
for(def c : citys){
def row = []
row << c
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
// for(def d : dates){
def startDate = "2022-07-01"
def count = findPgDealPrice(c, pgsqlInstance, startDate)
row << count*100/disCount
// def map = findSampleRate(pgsqlInstance, c, d)
// row << map.third
// }
rows << row
}
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
def findSampleRate(pgsqlInstance, cityName, yearmonth){
def sql = "SELECT urbn_nm,gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}"
def map = [:]
pgsqlInstance.eachRow(sql,{r->
map.deal = r.gujiamdltst_ppe15_pct
map.third = r.gujiatrdpttstppe15pct
});
return map
}
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
// cityRun(citys, startDate, endDate, cityCodeMap)
// regionRun(citys, startDate, endDate, cityCodeMap)
cityRun(citys, startDate, endDate, cityCodeMap)
regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
@@ -48,7 +48,7 @@ class HailanTest {
}
def run(){
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
def inputfile = "D:\\数据\\微元素\\排查表-价格对比-新.xlsx"
// addThirdPrice(inputfile)
// match(inputfile)
// tradeAvgPrice(inputfile)
@@ -58,7 +58,8 @@ class HailanTest {
// addHouse(inputfile)
// stat(inputfile)
// hailanStat(inputfile)
addFzFjwPrice(inputfile)
// addFzFjwPrice(inputfile)
addFjwTobPrice(inputfile)
}
def addHouse(inputfile){
@@ -748,6 +749,18 @@ class HailanTest {
})
}
def addFjwTobPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['F']){
def dbo = CommonDatautils.queryDistrictPrice("福州", d['F'], d['G'], DateUtils.parseFromTightByPattern("2023-07-02","yyyy-MM-dd"))
if(dbo){
d['Q'] = dbo.avg_price_2b
}
}
})
}
def addFzFjwPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['B']) {
@@ -252,7 +252,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "长滩美墅三期")
// q.put("name", "燃灯小区")
DBObject keys = new BasicDBObject();
if(fields){
@@ -250,8 +250,23 @@ class DataHandleUtils {
return sum;
}
public static isStandardRoad(def s){
def pattern = Pattern.compile("[路|街道|大道|巷|胡同|街|坊]\\d+(-\\d+)?(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
static main(args) {
def a = isStandardRoad("杨桥西路236号")
println a
}
}
@@ -5,7 +5,7 @@ import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.mongodb.BasicDBObject
//删除别名是另一个小区的主名
class MasterNameInAlias {
static main(args) {
@@ -14,7 +14,7 @@ class MasterNameInAlias {
}
def run(){
def citys = CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity().reverse()
def allList = []
for(def city : citys){
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
@@ -0,0 +1,616 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.sql.ResultSetMetaData
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AmapAddressUpdate {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
def task = new AmapAddressUpdate()
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
def addr = task.formatAddress("福州", "北三环路201号")
println addr
// task.run(path)
// task.similarMatch(path)
// task.addressRun(path)
// task.addressSource(path)
// task.amapRun()
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
// task.updateData(path)
// task.updateLocation(path)//更新经纬度
}
def run(path){
def citys = ["福州"]//CommonDatautils.findAllCity()
def allrets = []
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
for (def city : citys) {
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
def rets = findAmapDetail(disticts)
allrets.addAll(rets)
}
ExcelUtils.write(path, allrets, "数据")
}
def amapRun(){
def city = "福州市"
def datas = findAllAmapByCity(city)
// datas = datas[0..10]
println "Es总量:"+datas.size()
def rets = match(datas)
ExcelUtils.write("d:\\数据\\小区\\福州高德数据匹配.xlsx", rets, "数据")
}
def match(datas){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
rets << ["ID","高德城市","高德区域","高德名称","高德地址","匹配城市","匹配区域","匹配名称","匹配类型","匹配ID"]
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = []
def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
ret << d.id
ret << d.city
ret << d.region
ret << d.name
ret << d.address
if(jsonObject != null && jsonObject.precise!=-1) {
ret << jsonObject.get("city")
ret << jsonObject.get("district")
ret << jsonObject.get("name")
ret << jsonObject.get("propertyType")
ret << jsonObject.get("id")
}
println totalCount.get() +' '+ret
rets << ret
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findAllAmapByCity(city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT id,cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and type like '%住宅%'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def list = []
while (rs.next()) {
def map = [:]
map.id = rs.getString("id")
map.city = city
map.name = rs.getString("name")
map.address = rs.getString("address")
map.region = rs.getString("adname")
// map.type = rs.getString("type")
// def location = rs.getObject("location")
// if(location){
// map.lng = location.x
// map.lat = location.y
// }
list << map
}
return list
}
def findAmapDetail(districts){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def row = []
row << d.city
row << d.region
row << d.name
row << d.address
if(d.addr_alias){
row << d.addr_alias.join(" ")
}else{
row << ''
}
def dbo = getAmapLocation(d.city+"市", d.region, d.name)
if(!dbo){
def newName = d.name
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
}
dbo = getAmapLocation(d.city+"市", d.region, newName)
}
// if(!dbo && d.lat){
// dbo = getAmapLocation(d.city+"市", d.region, newName, d.lat/10000000000.0, d.lng/10000000000.0)
// }
if(dbo){
row << dbo.region
row << dbo.name
row << dbo.address
row << formatAddress(d.city, dbo.address)
row << dbo.lng
row << dbo.lat
row << dbo.match_type
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
row << ""
row << ""
}
println totalCount.get() +' '+row
rets << row
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def formatAddress(def city, def address){
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
def address_str = null
if(jsonObj && jsonObj.result) {
def street = ''
def street_number = ''
if (jsonObj.result.street) {
street = jsonObj.result.street
}
if (jsonObj.result.streetNumber) {
street_number = jsonObj.result.streetNumber
}
if (street || street_number) {
address_str = street
if(street_number){
address_str = address_str + street_number
}
}
}
if(address_str && address_str.contains("交叉口")){
address_str = address_str.substring(0, address_str.indexOf("交叉口")+3)
}
return address_str;
}
def getAmapLocation(city, region, name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and type like '%住宅%'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def map = [:]
while (rs.next()) {
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
def mapName = rs.getString("name").replace("·","")
if(mapName == name && rs.getString("adname").contains(region)){
map.name = rs.getString("name")
map.address = rs.getString("address")
map.formatAddress = AddressUtil.formatAddress(city, map.address)
map.region = rs.getString("adname")
map.type = rs.getString("type")
map.match_type = "名称相同"
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
}
}
// String sql = "select * from amap_poi_detail where cityname='${city}' and name='${name}' and type like '%住宅%'"
// def loc = null
// ElasticUtils.executeSqlQuery(sql,{ data->
// println data.adname+" "+data.name +" "+data.address +" "+data.type
// if(name == data.name && data.adname.contains(region) && data.type.contains("type")){
// data.formatAddress = AddressUtil.formatAddress(city, data.address)
// loc = data
// }
// })
return map;
}
def addressRun(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
// if(row['H'] && row['H'].contains("口步行") && row['H'].contains("(")){
// row['I'] = row['H'].substring(0, row['H'].indexOf("("))
// }
if(row['J']){
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
def distance = null
if(dist1){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, row['J'].toString().toDouble(), row['K'].toString().toDouble());
}
row['M'] = distance
}
def gaodeRoadFlag = false
if(row['I']){
gaodeRoadFlag = DataHandleUtils.isStandardRoad(row['I'])
}
def dbRoadFlag = false
if(row['D']){
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
}
if(!row['N'] && row['D'] && row['I']){
if(row['D']==row['I']){
row['N'] = 100
}else if(row['E'] && gaodeRoadFlag && row['E'].contains(row['I'])){
row['N'] = 1
}
}
if(!row['N'] && !dbRoadFlag && gaodeRoadFlag){
row['N'] = 2
}
if(!row['N'] && dbRoadFlag && gaodeRoadFlag && row['I'].contains(row['D'])){
row['N'] = 3
}
if(!row['N'] && dbRoadFlag && gaodeRoadFlag){
row['N'] = 4
}
})
}
def addressSource(inputfile) {
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(!row['N'] && row['D'] && !row['O']){
def addresses = findAllEsData(row['A'], row['B'], row['C'])
if(addresses){
def dbRoadFlag = false
if(row['D']){
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
}
if(dbRoadFlag){
def list = []
list << row['D']
if(row['E']){
def arr = row['E'].split(" ")
list.addAll(arr as List)
}
for(def addr : list){
if(addresses.contains(addr)){
row['O'] = "诸葛找房"
break
}
}
}
}
}
})
}
def findAllEsData(city,region, name){
def rets = []
def sql = "select * from community where city='${city}' and name='${name}' and source='诸葛找房APP'"
rets = findDatas(region, sql)
if(!rets){
rets = findAliasDatas(city,region, name)
}
return rets;
}
def findDatas(region, sql){
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
if(d.region && d.region.contains(region) && d.address){
rets << d.address
}
})
return rets;
}
def findAliasDatas(city, region, name){
def must = []
def term1 = ["match":["city":city]]
must << term1
def should = []
should << ["match":["alias":name]]
def params = ["query":["bool":["must":must, "should":should]]]
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/community/_doc/_search");
def rets = [];
if(s) {
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
hits = hits.hits
for(def h : hits){
def d = h._source
if(d.region && d.region.contains(region) && d.address && d.source=="诸葛找房APP"){
rets << d.address
}
}
}
}
return rets;
}
def addrAndLocationDistance(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(!row['N'] && row['D']){
println row['D']
def dbRoadFlag = false
if(row['D']){
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
}
if(dbRoadFlag){
def gloJson = GaoDeApiUtils.geo(row['A'], row['D'])
if(gloJson) {
def glocation = null
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def map = geocodes.get(0)
if(map.formatted_address.contains(row['D'])){
def arrs = map.location.split(",")
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
def distance = null
if(dist1){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, arrs[0].toString().toDouble(), arrs[1].toString().toDouble());
}
row['P'] = distance
}
}
}
}
}
})
}
def updateData(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['Q'] && row['Q'].contains("修改地址")){
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
def udata = new BasicDBObject()
if(row['Q'].contains("可替换原地址")){
udata.put("address", row['I'])
}else if(row['Q'].contains("可新增地址别名")){
udata.put("address", row['I'])
def addrAlias = dist1.addr_alias
if(!addrAlias){
addrAlias = []
}
addrAlias << dist1.address
udata.put("addr_alias", addrAlias)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata))
}
})
}
def updateLocation(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['R'] && row['R']=="是"){
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
if(dist1){
def udata = new BasicDBObject()
def lat = null
def lng = null
if(row['J']){
lng = row['J'].toString().toDouble()
lat = row['K'].toString().toDouble()
}else{
def gloJson = GaoDeApiUtils.geo(row['A'], row['D'])
if(gloJson) {
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def map = geocodes.get(0)
if (map.formatted_address.contains(row['D'])) {
def arrs = map.location.split(",")
lng = arrs[0].toString().toDouble()
lat = arrs[1].toString().toDouble()
}
}
}
}
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long lng_new = lng *10000000000L
Long lat_new = lat *10000000000L
udata.put("lng", lng_new)
udata.put("lat", lat_new)
if(lng2){
Long lng2_new = lng2 *10000000000L
Long lat2_new = lat2 *10000000000L
udata.put("lng2", lng2_new)
udata.put("lat2", lat2_new)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata))
}
}
})
}
def similarMatch(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def d = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
if(d && d.lat){
def dbo = getSimilarAmapLocation(d.city+"市", d.region, d.name, d.address, d.lat/10000000000.0, d.lng/10000000000.0)
if(dbo){
row['F'] = dbo.region
row['G'] = dbo.name
row['H'] = dbo.address
row['I'] = formatAddress(d.city, dbo.address)
row['J'] = dbo.lng
row['K'] = dbo.lat
row['L'] = dbo.score
row['M'] = dbo.distance
row['S'] = dbo.type
}
}
}
})
}
def getSimilarAmapLocation(city, region, name, address, lat, lng){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and MATCH(name,'${name}')")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def maxSocreObj = null
while (rs.next()) {
def map = [:]
def mapName = rs.getString("name").replace("·","")
println name +" "+ mapName
if(rs.getString("adname").contains(region)){
map.name = rs.getString("name")
map.address = rs.getString("address")
map.region = rs.getString("adname")
map.type = rs.getString("type")
map.match_type = "名称相似"
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
double nameScore = 0
double addScore = 0
double loctionScore = 0
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(lng, lat, map.lng.toString().toDouble(), map.lat.toString().toDouble());
map.distance = distance
if(map.name && name){
nameScore = CosineSimilarAlgorithm.getSimilarity(map.name, name);
}
if(map.address && address){
addScore = CosineSimilarAlgorithm.getSimilarity(map.address, address);
}
if(hasColumn(rs, "shape")){
def shape = rs.getObject("shape")
if(shape){
def coordinates = shape.getObject("coordinates")
// def pointList = InsidePolygon.createPoint2(points);
def flag = InsidePolygon.isInArea(lng, lat, coordinates)
if(flag){
loctionScore = 0.5
}
}
}else{
if(distance < 500 && (nameScore>0.5 || addScore>0.5)){
loctionScore = 0.3
}
}
def score = nameScore*0.3 + addScore*0.2 + loctionScore
map.score = score
if(!maxSocreObj){
if(score>0){
map.formatAddress = AddressUtil.formatAddress(city, map.address)
maxSocreObj = map
}
}else{
println score +" "+maxSocreObj.score
if(maxSocreObj && score>maxSocreObj.score){
map.formatAddress = AddressUtil.formatAddress(city, map.address)
maxSocreObj = map
}
}
}
}
return maxSocreObj;
}
def hasColumn(ResultSet rs, String columnName) {
ResultSetMetaData rsmd = rs.getMetaData();
int columns = rsmd.getColumnCount();
for (int x = 1; x <= columns; x++) {
if (columnName.equals(rsmd.getColumnName(x))) {
return true;
}
}
return false;
}
}
@@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
}
def calcMonthlyPrice(){
def citys = CommonUtils.findAllCity()
def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
@@ -2,6 +2,9 @@ package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.JsonUtils
import com.fangjia.core.util.Message
import com.fangjia.core.util.MessageInfo
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.util.DateUtils
@@ -9,26 +12,71 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
import com.fangjia.core.util.HttpRequest
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
def city = RegionData.formatCity("余姚")
d['D'] = city
if(city){
d['E'] = RegionData.formatRegion(city, d['C'])
}
// if(!d['D']){
// def citys = CommonDatautils.getCityByRegion(d['A'])
// if(citys && citys.size()==1){
// d['F'] = citys.get(0)
// }
//Message message = new Message();
//message.setMsgtype("text");
//message.setText(new MessageInfo("测试"));
//URL url = new URL("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430");
//// 建立http连接
//HttpURLConnection conn = (HttpURLConnection) url.openConnection();
//conn.setDoOutput(true);
//conn.setDoInput(true);
//conn.setUseCaches(false);
//conn.setRequestMethod("POST");
//conn.setRequestProperty("Charset", "UTF-8");
//conn.setRequestProperty("Content-Type", "application/Json; charset=UTF-8");
//conn.connect();
//OutputStream out = conn.getOutputStream();
//String textMessage = JsonUtils.toJson(message);
//byte[] data = textMessage.getBytes("UTF-8");
//out.write(data);
//out.flush();
//out.close();
//System.out.println(conn.getResponseCode());
//InputStream ins = conn.getInputStream();
//byte[] data1 = new byte[ins.available()];
//ins.read(data1);
//System.out.println(new String(data1));
//HttpRequest request = new HttpRequest()
//def params = [:]
//params.put("","")
//request.post("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430","", "utf-8")
//sendMarkdownMessage("租金", "测试")
//static void sendMarkdownMessage(String title, String msg){
// HttpRequest request = new HttpRequest()
// JSONObject requestBody = new JSONObject()
// JSONObject markdown = new JSONObject()
// markdown.put("title", title)
// markdown.put("text", msg)
// requestBody.put("markdown", markdown)
// requestBody.put("msgtype", "markdown")
// request.postWithBody("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430", requestBody, null, "UTF-8")
//}
//ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
// def city = RegionData.formatCity("余姚")
// d['D'] = city
// if(city){
// d['E'] = RegionData.formatRegion(city, d['C'])
// }
})
//
//// if(!d['D']){
//// def citys = CommonDatautils.getCityByRegion(d['A'])
//// if(citys && citys.size()==1){
//// d['F'] = citys.get(0)
//// }
//// }
//})
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
//def m = [:]
@@ -1,9 +1,14 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.HttpRequestUtil
import com.mongodb.DBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
@@ -153,4 +158,18 @@ class FangjiaClientUtils {
return columnValMap;
}
public static def saveOrUpdateDistrict(data) {
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
if (json.status == 0) {
return "成功"
}else if(json.status == 1){
return json.msg
}
}
}
@@ -10,6 +10,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
*处理地址在地址别名中有重复
*/
class SameDistName {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
@@ -18,13 +21,13 @@ class SameDistName {
def task = new SameDistName()
// task.run()
//地址和地址别名完全一致
// task.addressPocess()
task.addressPocess()
}
def addressPocess(){
def citys = CommonDatautils.findAllCity();
def citys = ["福州"]//CommonDatautils.findAllCity();
for(def c : citys){
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"])
@@ -0,0 +1,121 @@
package com.fangjia.script.gj.location
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 判断小区是否在城市范围内
*/
class DistrictLocationBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static DBCollection RANGECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
static main(args) {
def task = new DistrictLocationBatch()
// task.run()
task.deleteDist()
}
def deleteDist(){
ExcelUtils.multiThreadingModify("d:\\数据\\小区\\非本城市小区.xlsx",{d->
DBObject q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['B'])
q.put("name", d['C'])
q.put("cat", 'district')
q.put("status", 0)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
})
}
def run() {
def citys = CommonDatautils.findAllCity()
def cityMap = findCityRange()
def allrets = []
for (def city : citys) {
def points = cityMap.get(city)
def pointList = InsidePolygon.createPoint3(points);
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "lat", "lng"])
def rets = findNotInCityDistricts(city, disticts, pointList)
allrets.addAll(rets)
}
ExcelUtils.write("d:\\数据\\小区\\非本城市小区.xlsx", allrets, "经纬度不在此城市中")
}
def findNotInCityDistricts(city, districts, pointList){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(dist.lng){
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
if(!flag) {
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
ret << dist.address
println ret
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
println city+" "+ rets.size() +" "+districts.size()
return rets
}
def findCityRange(){
def q = new BasicDBObject()
def dbcursor = RANGECOLL.find(q)
def cityMap = [:]
for(def d : dbcursor){
cityMap.put(d.fjw_city, d.polyline)
}
return cityMap
}
def isInArea(lng, lat, pointList){
def flag = false;
for(def pointArr : pointList){
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
if(flag){
break;
}
}
return flag;
}
}
@@ -0,0 +1,84 @@
package com.fangjia.script.gj.location
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
class GaodeCityRange {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
static main(args) {
def task = new GaodeCityRange()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
HttpRequest request = new HttpRequest()
def rows = []
citys.each{c->
if(!citys1.contains(c)){
c = c.replace("州","")
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
if(resp.containsKey("districts")){
def datas = resp.get("districts")
for(def d : datas){
println c +" "+ d.name+" "+d.level
if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){
rows << d
}else{
if(d.level=="city"){
rows << d
}
}
}
}
}
}
// citys.each{c->
// println c
// JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
// if(resp.containsKey("districts")){
// def datas = resp.get("districts")
// for(def d : datas){
// println c +" "+ d.name+" "+d.level
// if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){
// rows << d
// }else{
// if(d.level=="city"){
// rows << d
// }
// }
// }
// }
// }
if(rows){
rows.each {r->
def obj = new BasicDBObject()
obj.put("level", r.level)
obj.put("center", r.center)
obj.put("center", r.center)
obj.put("polyline", r.polyline)
obj.put("name", r.name)
obj.put("adcode", r.adcode)
obj.put("citycode", r.citycode)
def city = RegionData.formatCity(r.name)
obj.put("fjw_city", city)
POLYLINECOLL.save(obj)
}
}
}
}
@@ -3,6 +3,9 @@ package com.fangjia.script.gj.location
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
@@ -25,28 +28,122 @@ import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern;
class LocationTest {
/**
* 根据区域围栏修正区域
*/
@BatchOwner(Watcher.GaoJuan)
class RegionLocationUpdateBatch extends BaseBatch{
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
// Date date=new Date(1622802609);
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// String str = format.format(date);
// System.out.println(str);
runBatch(new RegionLocationUpdateBatch(), args);
}
def task = new LocationTest()
task.run()
@Override
public void process(Object args) {
regionUpdate()
// task.run()
// task.run2()
// task.run3()
// task.run4()
// task.run5()
}
def regionUpdate(){
def citys = CommonDatautils.findAllCity()
def rets = []
def index = 0
for(def city : citys){
index ++
def regionPointsMap = findRegionRange(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
def datas = findInRegionDistricts(city, districts, regionPointsMap)
println "【${index}】"+city +" 经纬度处理完成"
rets.addAll(datas)
}
ExcelUtils.write("D:\\数据\\小区\\区域纠正.xlsx", rets, "数据")
}
def findInRegionDistricts(city, districts, regionPointsMap){
def index = 0
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def regionMap = [:]
regionPointsMap.each{k,v->
def pointList = InsidePolygon.createPoint3(v);
regionMap.put(k, pointList)
}
def keys = regionPointsMap.keySet()
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
if(dist.lng){
totalCount.addAndGet(1)
def pointList = regionMap.get(dist.region)
def flag = InsidePolygon.isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
def newRegion = null
if(!flag){
for(def m : regionPointsMap){
if(m.key != dist.region){
pointList = regionMap.get(m.key)
flag = InsidePolygon.isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
if(flag){
newRegion = m.key
break;
}
}
}
}
if(flag){
if(newRegion){
println dist.city +" "+dist.region+" "+dist.name+" 【${newRegion}】"
def udata = [:]
udata.put("region", newRegion)
udata.put("status", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
udata.put("id", dist._id.toString())
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
def opt = "update"
if(s && s!='成功'){
if(s && s.contains("该区域小区已存在")){
def delUdata = new BasicDBObject()
delUdata.put("status", 1)
delUdata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
saveOptLog(dist.city, dist.region, dist.name)
opt = "delete"
}
}
rets << [dist.city, dist.region, dist.name, newRegion, opt]
}
}else{
println dist.city +" "+dist.region+" "+dist.name+" 没有找到对应的区域"
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def run(){
def city = "º¼ÖÝ"
def regionPointsMap = findRegionRange(city)
@@ -446,12 +543,11 @@ class LocationTest {
def findRegionRange(city){
def q = new BasicDBObject()
q.put("city",city+"ÊÐ")
q.put("city",city)
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline)
regionMap.put(d.fjw_region, d.polyline)
}
return regionMap
}
@@ -525,4 +621,22 @@ class LocationTest {
return rets
}
def saveOptLog(city, region, name){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category","district");
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", "remove")
obj.put("user_name", "高娟")
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -1,25 +1,19 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.location
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.math.RandomUtils
import org.apache.commons.lang3.StringUtils
import org.bson.Document
import java.util.regex.Pattern
/**
* 抓取区域边框
* 高德抓取区域边框
*/
class RegionPointCrawl {
@@ -28,13 +22,19 @@ class RegionPointCrawl {
static main(args) {
def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
//4个直辖市 北京市、上海市、天津市、重庆市
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
def provinces = ["甘肃省"]
def index = 0
for(def p : provinces){
index ++
println index +"/"+provinces.size() +" "+p
task.crawlCityAndDistrictCode(p);
println "城市 区域 code 完成"
task.run(p);
println "区域围栏 完成"
}
}
@@ -43,8 +43,8 @@ class RegionPointCrawl {
crawlCity(province)
}
def run(city){
def cityCodes = findCityCode(city);
def run(pro){
def cityCodes = findCityCode(pro);
if(cityCodes){
for(def cityMap : cityCodes){
def codeMap = findAllDistrictCode(cityMap.city_code);
@@ -59,7 +59,8 @@ class RegionPointCrawl {
if(ret.size()>0){
def standardCity = RegionData.formatCity(cityMap.name)
if(standardCity){
saveData(standardCity, m.name, m.ad_code , ret);
def newRegion = RegionData.formatRegion(standardCity, m.name)
saveData(pro, standardCity, m.name, m.ad_code , ret, newRegion);
}else{
println cityMap.name +" ûÓбê×¼»¯"
}
@@ -102,7 +103,7 @@ class RegionPointCrawl {
def findCityCode(city){
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
def q = null;
if(city.contains("省") || city.endsWith("自治区")){
if(city.contains("省") || city.endsWith("自治区") || ["上海市","天津市","北京市","重庆市"].contains(city)){
q = new BasicDBObject("province", pattern);
}else{
q = new BasicDBObject("name", pattern);
@@ -156,18 +157,22 @@ class RegionPointCrawl {
}
}
def saveData(city, region, code, ret){
def saveData(pro, city, region, code, ret, formatRegion){
def q = new BasicDBObject()
q.put("code", code)
def dbo = POLYLINECOLL.findOne(q)
if(!dbo){
def o = new BasicDBObject()
o.put("province", pro)
o.put("city", city)
o.put("region", region)
o.put("code", code)
o.put("center", ret.center.get(0))
o.put("level", ret.level.get(0))
o.put("polyline", ret.polyline.get(0));
if(formatRegion){
o.put("fjw_region", formatRegion)
}
POLYLINECOLL.save(o);
}
}
@@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
//广州,佛山 重庆
@@ -204,8 +204,11 @@ public class BeiKeCrawlByChrome {
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
def dbo = findTrade(durl , "贝壳找房")
if (!dbo) {
parse(city, e, region, driver)
parse(city, e, region, driver, null)
}else if(dbo && !dbo.lift){
parse(city, e, region, driver, dbo._id)
}
}else{
stop = true;
@@ -224,7 +227,7 @@ public class BeiKeCrawlByChrome {
return detailUrls
}
def parse(city,e, region, driver){
def parse(city,e, region, driver, id){
try {
def detailUrl = e.select(".title a").attr("href")
@@ -329,8 +332,11 @@ public class BeiKeCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
if(id){
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
}else{
DEAL_PRICE_COLL.save(detail)
}
}
}
}catch(Exception ex){
@@ -459,9 +465,9 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
// if (citys.contains(e.text())){
if (citys.contains(e.text())){
list << [e.text(), href]
// }
}
}
}
return list
@@ -33,7 +33,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -303,7 +303,9 @@ public class LianjiaCrawlByChrome {
detailUrls << durl
def dbo = findTrade(durl , "Á´¼ÒÔÚÏß")
if (!dbo) {
parse(city, e, regionText, driver)
parse(city, e, regionText, driver, null)
}else if(dbo && !dbo.lift){
parse(city, e, regionText, driver, dbo._id)
}
}else{
stop = true;
@@ -326,7 +328,7 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def parse(city,e, regionText, driver){
def parse(city,e, regionText, driver, id){
try {
def price = e.select(".totalPrice .number").text()
if(price){
@@ -433,7 +435,12 @@ public class LianjiaCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
if(id){
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
}else{
DEAL_PRICE_COLL.save(detail)
}
}
}
}
@@ -563,9 +570,9 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href]
// }
}
}
}
return cityList
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -21,10 +22,11 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
}
@@ -105,12 +107,12 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['B']) {
// def dist = CommonDatautils.findDistrictById(d['B'])
// if(dist){
// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['G'] = gxdPrice
// }
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['I'] = gxdPrice
}
}
})
}
@@ -129,4 +131,22 @@ class AddPriceTest {
}
return price
}
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd"))
if(dbo){
// d['K'] = dbo.avg_price
d['K'] = dbo.avg_price_2b
}
}
}
})
}
}
@@ -294,4 +294,8 @@ class AddressUtil {
return result
}
public static void main(String[] args){
def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区")
}
}
@@ -52,7 +52,40 @@ class InsidePolygon {
return list;
}
public static List<Point> createPoint3(String points){
List<Point> alllist = new ArrayList<Point>();
if(points){
List<String[]> arrs = new ArrayList<String[]>();
if(points.contains("|")){
String[] array = points.split("\\|");
arrs.addAll(array)
}else{
arrs.add(points)
}
for(def arrStr : arrs){
List<Point> list = new ArrayList<Point>();
String[] array = arrStr.split(";");
for(int i =0 ;i<array.length-1;i++){
def pointArrs = array[i].split("[,|\\|]")
Point point = new Point();
double x = Double.parseDouble(pointArrs[0]);
double y = Double.parseDouble(pointArrs[1]);
point.setLon(x);
point.setLat(y);
list.add(point);
}
alllist << list
}
}
return alllist;
}
public static List<Map<String, Object>> getPoints(String points){
String[] array = points.split("-");
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
@@ -227,4 +260,16 @@ class InsidePolygon {
}
return false;
}
public static def isInArea(lng, lat, pointList){
def flag = false;
for(def pointArr : pointList){
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
if(flag){
break;
}
}
return flag;
}
}
@@ -1,7 +1,9 @@
package com.fangjia.script.temp.shenjian.util
import net.sf.json.JSONObject
import org.apache.commons.httpclient.HttpClient
import org.apache.commons.httpclient.methods.PostMethod
import org.apache.commons.httpclient.methods.RequestEntity
import org.apache.commons.httpclient.methods.StringRequestEntity
/**
@@ -26,4 +28,42 @@ class HttpUtils {
}
return false
}
static def doPost2(String url, String jsonString){
HttpClient httpClient = new HttpClient()
PostMethod postMethod = new PostMethod(url)
String responseMsg = null;
try{
StringRequestEntity requestEntity = new StringRequestEntity(
jsonString,
"application/json",
"UTF-8");
postMethod.setRequestEntity(requestEntity);
httpClient.executeMethod(postMethod);
InputStream is = postMethod.getResponseBodyAsStream();
BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
StringBuffer sb = new StringBuffer("");
String str= "";
while((str = br.readLine()) != null){
sb.append(str);
}
responseMsg = sb.toString();
} catch (Exception ex) {
ex.printStackTrace();
} finally {
postMethod.releaseConnection();
}
return JSONObject.fromObject(responseMsg);
}
public String encodeHtml(String html) {
String mh=html;
try {
mh=new String(html.getBytes("iso-8859-1"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
return mh;
}
}
@@ -28,7 +28,25 @@ class TradeDealPriceStat {
static main(args) {
def task = new TradeDealPriceStat()
task.run()
// task.run()
task.deleteRepeat()
}
def deleteRepeat(){
def q = new BasicDBObject()
q.put("source", "ÎÒ°®ÎÒ¼Ò")
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def set = new HashSet()
for(def c : cursor){
def tradeDate = DateUtils.formatDateByPattern(c.trade_date, "yyyyMMdd")
def key = c.city +"#"+c.region+"#"+c.district_name+"#"+c.avg_price+"#"+tradeDate
if(set.contains(key)){
DEAL_PRICE.remove(new BasicDBObject('_id', c._id))
println c._id
}else{
set << key
}
}
}
def run(){
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303")
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20221001", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230301", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0