Merge branch 'script_gj' into 'master'

Script gj

See merge request !77
This commit is contained in:
2021-08-07 19:52:57 +08:00
7 changed files with 191 additions and 64 deletions
@@ -148,11 +148,11 @@ class BigdataCommunityBatch extends BaseBatch{
ret << "pa100"
ret << ""
if(dist.address){
if(dist.address.toString().endsWith("弄")){
if(dist.address.toString().contains("弄")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << number
ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
}else{
ret << ""
ret << ""
@@ -393,14 +393,14 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
def startDate = null;
if(citys.contains(city)){
// if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
}else{
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
}
// }else{
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
@@ -2,15 +2,16 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def citys = CcbCommonUtils.citys;
//def rows = []
//
@@ -25,8 +26,8 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
// println city +" "+count
//}
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0802(3).xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\成交统计.xlsx",0,1)
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
// dealMap.put(d['A'], d['B'])
@@ -35,52 +36,99 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
//
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['C']){
// d['P'] = dealMap.get(d['C'])
// d['Q'] = dealMap.get(d['C'])
// }
//},2)
def city = "中山"
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
def index =0
def updateIndex = 0
def delIndex = 0
for(def dist : dists){
if(dist.region=="石岐区街道"){
def lat = dist.lat/10000000000.0
def lng = dist.lng/10000000000.0
def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
if(regionStr){
def region = RegionData.formatRegion(dist.city, regionStr)
def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
if(obj){
delIndex++
DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1)))
println delIndex+" [del]"+region +" "+dist.name
}else{
updateIndex++
DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
println updateIndex +" [update]"+region+" "+dist.name
}
}else{
index++
println index+" [没找到]"+dist.name
}
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
// def map = [:]
// map.put("land_area",d['D'])
// map.put("build_area",d['F'])
// map.put("greeningrate_area",d['H'])
// map.put("parkingcnt_area",d['J'])
// map.put("zts_area",d['L'])
// map.put("buildyear_area",d['N'])
// map.put("plotratio",d['P'])
//
// dealMap.put(d['A'], map)
//}
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(fileName,{d->
def city = d['B']
if(city){
def cityid = cityCodeDatas.get(city)
def total = findCount(cityid, sqlInstance)
d['I'] = total
// def a = findEstateType(cityid, sqlInstance)
//
// d['AA'] = total
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
//
// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
//
// def c = findFiledCount(cityid, 'address', sqlInstance)
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
//
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
//
// def e = findFiledCount(cityid, 'landarea', sqlInstance)
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
//
// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
//
// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
//
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
//
// def i = findFiledCount(cityid, 'latitude', sqlInstance)
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
//
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
println d
}
},0)
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def getStreetInfo(def city, def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent && addressComponent.city.contains(city)){
township = addressComponent.township
}
}
}
return township;
def findCount(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findFiledCount(cityid, field, sqlInstance){
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
@@ -88,6 +136,54 @@ def getStreetInfo(def city, def glocation){
//def city = "中山"
//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
//def index =0
//def updateIndex = 0
//def delIndex = 0
//for(def dist : dists){
// if(dist.region=="石岐区街道"){
// def lat = dist.lat/10000000000.0
// def lng = dist.lng/10000000000.0
// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
// if(regionStr){
// def region = RegionData.formatRegion(dist.city, regionStr)
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){
// delIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1)))
// println delIndex+" [del]"+region +" "+dist.name
// }else{
// updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]"+region+" "+dist.name
// }
// }else{
// index++
// println index+" [没找到]"+dist.name
// }
// }
//}
//
//def getStreetInfo(def city, def glocation){
// def township = null
// if(glocation){
// def jsonObject = GaoDeApiUtils.geocode(glocation)
// if(jsonObject && jsonObject.regeocode) {
// def addressComponent = jsonObject.regeocode.addressComponent
// if(addressComponent && addressComponent.city.contains(city)){
// township = addressComponent.township
// }
// }
// }
// return township;
//}
@@ -780,6 +780,8 @@ class CommonDatautils {
keys.put("address", 1);
keys.put("build_structure", 1);
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -29,7 +29,7 @@ import java.util.regex.Pattern;
class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -40,9 +40,9 @@ class LocationTest {
// System.out.println(str);
def task = new LocationTest()
task.run()
// task.run()
// task.run2()
// task.run3()
task.run3()
// task.run4()
// task.run5()
}
@@ -162,7 +162,9 @@ class LocationTest {
def run3(){
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d->
def index = 0
ExcelUtils.modify("d:\\数据\\福州\\经纬度错误.xlsx", {d->
index++
def distance = null
if(d['H']){
distance = d['H'].toDouble()
@@ -261,6 +263,7 @@ class LocationTest {
}
}
}
println index
},0)
}
@@ -311,7 +314,7 @@ class LocationTest {
}
}
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
def baiduRest = null;//FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
if(baiduRest){
bdlocationMap = getMapLocation(dist, baiduRest);
if(bdlocationMap){
@@ -464,7 +467,11 @@ class LocationTest {
void run() {
try{
totalCount.addAndGet(1)
mapLocation(map)
def ret = mapLocation(map)
if(ret){
rets << ret
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -474,14 +481,35 @@ class LocationTest {
});
}
latch.await()
return rets;
ExcelUtils.write("d:\\数据\\福州\\经纬度错误.xlsx", rets, "经纬度错误")
}
def mapLocation(map){
def dist = CommonDatautils.findDistrictById(map.commid)
def dist = CommonDatautils.findDistrictById(map[1])
if(dist){
def loactionMap = matchMap(dist)
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
if (loactionMap) {
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if (dist.region == loactionMap.region) {
println dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
} else {
println dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
}
}
return ret
}
return null
}
@@ -494,10 +522,7 @@ class LocationTest {
sqlInstance.eachRow(sql,{r->
rets << [r.cityid, r.commid]
});
def map = [:]
return map
return rets
}
}
@@ -90,7 +90,7 @@ class AddressStandard {
}
}
}
},0)
},0, "Sheet1")
}
@@ -60,7 +60,8 @@ public class ExcelUtils{
FileInputStream inp = null;
try {
inp = new FileInputStream(file);
Workbook workbook = WorkbookFactory.create(inp)
BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 200)
Workbook workbook = WorkbookFactory.create(bufferedInputStream)
workbook.setForceFormulaRecalculation(true)
Sheet sheet = workbook.getSheetAt(sheetIndex);
//行号
@@ -82,6 +83,9 @@ public class ExcelUtils{
}
}
ret.add(rowData);
if (ret.size() % 10000 == 0) {
println(ret.size())
}
}
} catch (IOException e) {
e.printStackTrace();