updadte
This commit is contained in:
+138
-5
@@ -1,7 +1,12 @@
|
|||||||
package com.fangjia.script.gj.data_handle
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.util.ElasticUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.GeoHelper
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
import java.util.concurrent.CopyOnWriteArrayList
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
import java.util.concurrent.CountDownLatch
|
import java.util.concurrent.CountDownLatch
|
||||||
@@ -9,17 +14,79 @@ import java.util.concurrent.ExecutorService
|
|||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.atomic.AtomicInteger
|
import java.util.concurrent.atomic.AtomicInteger
|
||||||
import java.util.concurrent.atomic.AtomicLong
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
class TempExportData {
|
class TempExportData {
|
||||||
|
|
||||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||||
|
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||||
|
DBCollection qianzhan_coll = MongoUtils.getCollectionByName("crawl", "yuanqu_qianzhan_com_map", "192.168.0.206", 27020)
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new TempExportData()
|
def task = new TempExportData()
|
||||||
// task.run()
|
// task.run()
|
||||||
def rets = task.fzHouseStat()
|
// def rets = task.fzHouseStat()
|
||||||
ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
|
// ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
|
||||||
println "文件已生成"
|
// println "文件已生成"
|
||||||
|
// exportDistShape()
|
||||||
|
// task.exportDistrictAlias()
|
||||||
|
task.exportOfficeHouseCount()
|
||||||
|
}
|
||||||
|
|
||||||
|
def exportDistShape(){
|
||||||
|
def task = new BasicDBObject()
|
||||||
|
}
|
||||||
|
|
||||||
|
def exportDistrictAlias(){
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
def allRets = []
|
||||||
|
allRets << ["Id","城市","区域","名称","地址","已删ID","已删区域","已删名称","已删地址","距离"]
|
||||||
|
|
||||||
|
for (def city : citys) {
|
||||||
|
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "lat","lng"])
|
||||||
|
def rets = findDelDistBatch(dists)
|
||||||
|
allRets.addAll(rets)
|
||||||
|
}
|
||||||
|
ExcelUtils.write("D:\\数据\\数据建设\\小区\\融合存疑小区.xlsx",allRets, "小区")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findDelDistBatch(dists){
|
||||||
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
for(def i=0;i<dists.size();i++){
|
||||||
|
final def dist = dists[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
def nameAlias = dist.alias
|
||||||
|
if(nameAlias){
|
||||||
|
for(def nameAlia : nameAlias){
|
||||||
|
def delDists = CommonDatautils.findAllStatusDistrict(dist.city, dist.region, nameAlia, "district", ["city", "region", "name", "address", "alias", "lat","lng"])
|
||||||
|
if(delDists){
|
||||||
|
if(delDists.size()==1){
|
||||||
|
def delDist = delDists[0]
|
||||||
|
def distance = "缺经纬度"
|
||||||
|
if(delDist.lat){
|
||||||
|
distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, delDist.lat/10000000000.0, delDist.lng/10000000000.0)
|
||||||
|
}
|
||||||
|
println totalCount.get()+" "+dists.size() +" "+dist.city
|
||||||
|
rets << [dist._id.toString(), dist.city, dist.region, dist.name, dist.address,delDist._id.toString(), delDist.region, delDist.name, delDist.address, distance]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
return rets;
|
||||||
}
|
}
|
||||||
|
|
||||||
def run() {
|
def run() {
|
||||||
@@ -33,6 +100,8 @@ class TempExportData {
|
|||||||
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
|
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def getResults(city) {
|
def getResults(city) {
|
||||||
def rets = []
|
def rets = []
|
||||||
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
|
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
|
||||||
@@ -120,4 +189,68 @@ class TempExportData {
|
|||||||
return rets
|
return rets
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def findHouseCountByCat(def city, def region, def name, def cat){
|
||||||
|
|
||||||
|
String sql = "select count(*) as c from seawater where cat='${cat}' and city='${city}' and district_name='${name}' "
|
||||||
|
if(region){
|
||||||
|
sql = sql + " and region='${region}'"
|
||||||
|
}
|
||||||
|
def total = 0
|
||||||
|
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||||
|
total = data.c
|
||||||
|
})
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
def findYuanQu(city, name){
|
||||||
|
def q = new BasicDBObject("name", Pattern.compile("^${name}"))
|
||||||
|
def dbcursor = qianzhan_coll.find(q)
|
||||||
|
def obj = null
|
||||||
|
for(def d : dbcursor){
|
||||||
|
if(d.city.contains(city)){
|
||||||
|
obj = d
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return obj
|
||||||
|
}
|
||||||
|
|
||||||
|
def exportOfficeHouseCount(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d->
|
||||||
|
// def obj = findYuanQu(d['B'], d['D'])
|
||||||
|
// if(obj){
|
||||||
|
// d['T'] = "前瞻存在"
|
||||||
|
// }
|
||||||
|
// if(!d['T']){
|
||||||
|
// def obj = findYuanQu(d['B'], d['D'])
|
||||||
|
// if(obj){
|
||||||
|
// d['U'] = obj.name
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// if(!d['T']) {
|
||||||
|
// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell")
|
||||||
|
// if (!count) {
|
||||||
|
// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent")
|
||||||
|
// }
|
||||||
|
// d['S'] = count
|
||||||
|
// }
|
||||||
|
|
||||||
|
if(!d['J']){
|
||||||
|
def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D'])
|
||||||
|
if(datas){
|
||||||
|
def sources = []
|
||||||
|
for(def o : datas){
|
||||||
|
sources << o.source
|
||||||
|
}
|
||||||
|
if(sources){
|
||||||
|
d['K'] = sources.join(",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user