Merge branch 'script_gj' into 'master'

小区导出位置修改

See merge request !76
This commit is contained in:
2021-08-07 13:55:34 +08:00
3 changed files with 112 additions and 30 deletions
@@ -53,10 +53,15 @@ class BigdataCommunityBatch extends BaseBatch{
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
} }
def fileName = "/home/goojia/ccb/bigdata_community.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt"
}
def allDatas = [] def allDatas = []
allDatas << headers allDatas << headers
allDatas.addAll(rows) allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false); FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成" println "txt 生成"
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
} }
@@ -76,6 +76,7 @@ class TradeStatLog {
println ret println ret
} }
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据") ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据")
println "表格生成完成" println "表格生成完成"
} }
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point import com.fangjia.script.looplineinfo.Point
@@ -14,7 +15,13 @@ import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import groovy.sql.Sql
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher import java.util.regex.Matcher
import java.util.regex.Pattern; import java.util.regex.Pattern;
@@ -22,7 +29,10 @@ import java.util.regex.Pattern;
class LocationTest { class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) { static main(args) {
// Date date=new Date(1622802609); // Date date=new Date(1622802609);
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
@@ -34,6 +44,7 @@ class LocationTest {
// task.run2() // task.run2()
// task.run3() // task.run3()
// task.run4() // task.run4()
// task.run5()
} }
def run(){ def run(){
@@ -41,40 +52,57 @@ class LocationTest {
def regionPointsMap = findRegionRange(city) def regionPointsMap = findRegionRange(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]); def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
def index = 0 def index = 0
def rets = [] def rets = Collections.synchronizedList(new ArrayList());
for(def dist : districts){ def totalCount = new AtomicLong()
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) final CountDownLatch latch = new CountDownLatch(districts.size())
if(!flag){
index++
def ret = [] for(def i=0;i<districts.size();i++){
ret << dist.city final def dist = districts[i]
ret << dist.region
ret << dist.name
def loactionMap = matchMap(dist) fixedThreadPool.execute(new Runnable() {
if(loactionMap){ void run() {
ret << loactionMap.source try{
ret << loactionMap.region totalCount.addAndGet(1)
ret << loactionMap.mapName def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
ret << loactionMap.address if(!flag) {
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng) index++
ret << distance def ret = []
ret << loactionMap.lng ret << dist.city
ret << loactionMap.lat ret << dist.region
if(dist.region==loactionMap.region){ ret << dist.name
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
}else{ def loactionMap = matchMap(dist)
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance if (loactionMap) {
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if (dist.region == loactionMap.region) {
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
} else {
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
}
} else {
ret << ""
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内,地图没找到"
}
if (ret) {
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
} }
}else{
ret << ""
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
} }
rets << ret });
}
} }
latch.await()
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内") ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内")
} }
@@ -424,4 +452,52 @@ class LocationTest {
} }
return regionMap return regionMap
} }
def run5(){
def results = getAllErrorDistricts()
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
for(def i=0;i<results.size();i++){
final def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
mapLocation(map)
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def mapLocation(map){
def dist = CommonDatautils.findDistrictById(map.commid)
if(dist){
}
}
def getAllErrorDistricts(){
def sql = "select cityid,commid from bigdata_community_distcheck where distname<>checkdist or checkdist is null";
def rets = []
sqlInstance.eachRow(sql,{r->
rets << [r.cityid, r.commid]
});
def map = [:]
return map
}
} }