Merge branch 'script_gj' into 'master'

Script gj

See merge request !77
This commit is contained in:
2021-08-07 19:52:57 +08:00
7 changed files with 191 additions and 64 deletions
@@ -148,11 +148,11 @@ class BigdataCommunityBatch extends BaseBatch{
ret << "pa100" ret << "pa100"
ret << "" ret << ""
if(dist.address){ if(dist.address){
if(dist.address.toString().endsWith("弄")){ if(dist.address.toString().contains("弄")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄")) String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
if(number){ if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number)) ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << number ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
}else{ }else{
ret << "" ret << ""
ret << "" ret << ""
@@ -393,14 +393,14 @@ class BigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size()) final CountDownLatch latch = new CountDownLatch(dists.size())
def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] // def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
def startDate = null; def startDate = null;
if(citys.contains(city)){ // if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd")) startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
}else{ // }else{
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
} // }
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){ for(def i=0;i<dists.size();i++){
@@ -2,15 +2,16 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import groovy.sql.Sql import groovy.sql.Sql
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def citys = CcbCommonUtils.citys; //def citys = CcbCommonUtils.citys;
//def rows = [] //def rows = []
// //
@@ -25,8 +26,8 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
// println city +" "+count // println city +" "+count
//} //}
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0802(3).xlsx" //def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\成交统计.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
//def dealMap = [:] //def dealMap = [:]
//for(def d : datas){ //for(def d : datas){
// dealMap.put(d['A'], d['B']) // dealMap.put(d['A'], d['B'])
@@ -35,52 +36,99 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
// //
//ExcelUtils.multiThreadingModify(fileName,{d-> //ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['C']){ // if(d['C']){
// d['P'] = dealMap.get(d['C']) // d['Q'] = dealMap.get(d['C'])
// } // }
//},2) //},2)
def city = "中山"
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
def index =0 def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
def updateIndex = 0 //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
def delIndex = 0 //def dealMap = [:]
for(def dist : dists){ //for(def d : datas){
if(dist.region=="石岐区街道"){ // def map = [:]
def lat = dist.lat/10000000000.0 // map.put("land_area",d['D'])
def lng = dist.lng/10000000000.0 // map.put("build_area",d['F'])
def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString()) // map.put("greeningrate_area",d['H'])
if(regionStr){ // map.put("parkingcnt_area",d['J'])
def region = RegionData.formatRegion(dist.city, regionStr) // map.put("zts_area",d['L'])
def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district") // map.put("buildyear_area",d['N'])
if(obj){ // map.put("plotratio",d['P'])
delIndex++ //
DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1))) // dealMap.put(d['A'], map)
println delIndex+" [del]"+region +" "+dist.name //}
}else{
updateIndex++ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) ExcelUtils.multiThreadingModify(fileName,{d->
println updateIndex +" [update]"+region+" "+dist.name def city = d['B']
} if(city){
}else{ def cityid = cityCodeDatas.get(city)
index++ def total = findCount(cityid, sqlInstance)
println index+" [没找到]"+dist.name d['I'] = total
} // def a = findEstateType(cityid, sqlInstance)
//
// d['AA'] = total
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
//
// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
//
// def c = findFiledCount(cityid, 'address', sqlInstance)
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
//
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
//
// def e = findFiledCount(cityid, 'landarea', sqlInstance)
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
//
// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
//
// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
//
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
//
// def i = findFiledCount(cityid, 'latitude', sqlInstance)
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
//
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
println d
} }
},0)
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
} }
def getStreetInfo(def city, def glocation){ def findCount(cityid, sqlInstance){
def township = null def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'";
if(glocation){ def count = 0
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) { sqlInstance.eachRow(sql,{r->
def addressComponent = jsonObject.regeocode.addressComponent count = r.count
if(addressComponent && addressComponent.city.contains(city)){ });
township = addressComponent.township return count
} }
}
} def findFiledCount(cityid, field, sqlInstance){
return township; def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
} }
@@ -88,6 +136,54 @@ def getStreetInfo(def city, def glocation){
//def city = "中山"
//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
//def index =0
//def updateIndex = 0
//def delIndex = 0
//for(def dist : dists){
// if(dist.region=="石岐区街道"){
// def lat = dist.lat/10000000000.0
// def lng = dist.lng/10000000000.0
// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
// if(regionStr){
// def region = RegionData.formatRegion(dist.city, regionStr)
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){
// delIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1)))
// println delIndex+" [del]"+region +" "+dist.name
// }else{
// updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]"+region+" "+dist.name
// }
// }else{
// index++
// println index+" [没找到]"+dist.name
// }
// }
//}
//
//def getStreetInfo(def city, def glocation){
// def township = null
// if(glocation){
// def jsonObject = GaoDeApiUtils.geocode(glocation)
// if(jsonObject && jsonObject.regeocode) {
// def addressComponent = jsonObject.regeocode.addressComponent
// if(addressComponent && addressComponent.city.contains(city)){
// township = addressComponent.township
// }
// }
// }
// return township;
//}
@@ -780,6 +780,8 @@ class CommonDatautils {
keys.put("address", 1); keys.put("address", 1);
keys.put("build_structure", 1); keys.put("build_structure", 1);
keys.put("out_pics", 1); keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys); DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){ if(cursor){
for(def obj : cursor){ for(def obj : cursor){
@@ -29,7 +29,7 @@ import java.util.regex.Pattern;
class LocationTest { class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -40,9 +40,9 @@ class LocationTest {
// System.out.println(str); // System.out.println(str);
def task = new LocationTest() def task = new LocationTest()
task.run() // task.run()
// task.run2() // task.run2()
// task.run3() task.run3()
// task.run4() // task.run4()
// task.run5() // task.run5()
} }
@@ -162,7 +162,9 @@ class LocationTest {
def run3(){ def run3(){
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d-> def index = 0
ExcelUtils.modify("d:\\数据\\福州\\经纬度错误.xlsx", {d->
index++
def distance = null def distance = null
if(d['H']){ if(d['H']){
distance = d['H'].toDouble() distance = d['H'].toDouble()
@@ -261,6 +263,7 @@ class LocationTest {
} }
} }
} }
println index
},0) },0)
} }
@@ -311,7 +314,7 @@ class LocationTest {
} }
} }
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") def baiduRest = null;//FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
if(baiduRest){ if(baiduRest){
bdlocationMap = getMapLocation(dist, baiduRest); bdlocationMap = getMapLocation(dist, baiduRest);
if(bdlocationMap){ if(bdlocationMap){
@@ -464,7 +467,11 @@ class LocationTest {
void run() { void run() {
try{ try{
totalCount.addAndGet(1) totalCount.addAndGet(1)
mapLocation(map) def ret = mapLocation(map)
if(ret){
rets << ret
}
println totalCount.get()
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace() e.printStackTrace()
} finally { } finally {
@@ -474,14 +481,35 @@ class LocationTest {
}); });
} }
latch.await() latch.await()
return rets; ExcelUtils.write("d:\\数据\\福州\\经纬度错误.xlsx", rets, "经纬度错误")
} }
def mapLocation(map){ def mapLocation(map){
def dist = CommonDatautils.findDistrictById(map.commid) def dist = CommonDatautils.findDistrictById(map[1])
if(dist){ if(dist){
def loactionMap = matchMap(dist)
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
if (loactionMap) {
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if (dist.region == loactionMap.region) {
println dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
} else {
println dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
}
}
return ret
} }
return null
} }
@@ -494,10 +522,7 @@ class LocationTest {
sqlInstance.eachRow(sql,{r-> sqlInstance.eachRow(sql,{r->
rets << [r.cityid, r.commid] rets << [r.cityid, r.commid]
}); });
return rets
def map = [:]
return map
} }
} }
@@ -90,7 +90,7 @@ class AddressStandard {
} }
} }
} }
},0) },0, "Sheet1")
} }
@@ -60,7 +60,8 @@ public class ExcelUtils{
FileInputStream inp = null; FileInputStream inp = null;
try { try {
inp = new FileInputStream(file); inp = new FileInputStream(file);
Workbook workbook = WorkbookFactory.create(inp) BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 200)
Workbook workbook = WorkbookFactory.create(bufferedInputStream)
workbook.setForceFormulaRecalculation(true) workbook.setForceFormulaRecalculation(true)
Sheet sheet = workbook.getSheetAt(sheetIndex); Sheet sheet = workbook.getSheetAt(sheetIndex);
//行号 //行号
@@ -82,6 +83,9 @@ public class ExcelUtils{
} }
} }
ret.add(rowData); ret.add(rowData);
if (ret.size() % 10000 == 0) {
println(ret.size())
}
} }
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();