Merge branch 'script_gj' into 'master'

updadte

See merge request !1093
This commit is contained in:
2025-03-20 10:55:27 +08:00
@@ -1,7 +1,12 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -9,17 +14,79 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class TempExportData {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection qianzhan_coll = MongoUtils.getCollectionByName("crawl", "yuanqu_qianzhan_com_map", "192.168.0.206", 27020)
static main(args) {
def task = new TempExportData()
// task.run()
def rets = task.fzHouseStat()
ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
println "文件已生成"
// def rets = task.fzHouseStat()
// ExcelUtils.write("/home/goojia/gj/福州小区房源及第三方情况统计.xlsx", rets,"数据")
// println "文件已生成"
// exportDistShape()
// task.exportDistrictAlias()
task.exportOfficeHouseCount()
}
def exportDistShape(){
def task = new BasicDBObject()
}
def exportDistrictAlias(){
def citys = CommonDatautils.findAllCity()
def allRets = []
allRets << ["Id","城市","区域","名称","地址","已删ID","已删区域","已删名称","已删地址","距离"]
for (def city : citys) {
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "lat","lng"])
def rets = findDelDistBatch(dists)
allRets.addAll(rets)
}
ExcelUtils.write("D:\\数据\\数据建设\\小区\\融合存疑小区.xlsx",allRets, "小区")
}
def findDelDistBatch(dists){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def nameAlias = dist.alias
if(nameAlias){
for(def nameAlia : nameAlias){
def delDists = CommonDatautils.findAllStatusDistrict(dist.city, dist.region, nameAlia, "district", ["city", "region", "name", "address", "alias", "lat","lng"])
if(delDists){
if(delDists.size()==1){
def delDist = delDists[0]
def distance = "缺经纬度"
if(delDist.lat){
distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, delDist.lat/10000000000.0, delDist.lng/10000000000.0)
}
println totalCount.get()+" "+dists.size() +" "+dist.city
rets << [dist._id.toString(), dist.city, dist.region, dist.name, dist.address,delDist._id.toString(), delDist.region, delDist.name, delDist.address, distance]
}
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def run() {
@@ -33,6 +100,8 @@ class TempExportData {
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
}
def getResults(city) {
def rets = []
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
@@ -120,4 +189,68 @@ class TempExportData {
return rets
}
def findHouseCountByCat(def city, def region, def name, def cat){
String sql = "select count(*) as c from seawater where cat='${cat}' and city='${city}' and district_name='${name}' "
if(region){
sql = sql + " and region='${region}'"
}
def total = 0
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
return total
}
def findYuanQu(city, name){
def q = new BasicDBObject("name", Pattern.compile("^${name}"))
def dbcursor = qianzhan_coll.find(q)
def obj = null
for(def d : dbcursor){
if(d.city.contains(city)){
obj = d
break;
}
}
return obj
}
def exportOfficeHouseCount(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d->
// def obj = findYuanQu(d['B'], d['D'])
// if(obj){
// d['T'] = "前瞻存在"
// }
// if(!d['T']){
// def obj = findYuanQu(d['B'], d['D'])
// if(obj){
// d['U'] = obj.name
// }
// }
// if(!d['T']) {
// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell")
// if (!count) {
// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent")
// }
// d['S'] = count
// }
if(!d['J']){
def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D'])
if(datas){
def sources = []
for(def o : datas){
sources << o.source
}
if(sources){
d['K'] = sources.join(",")
}
}
}
})
}
}