+2
-2
@@ -148,11 +148,11 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
ret << "pa100"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().endsWith("弄")){
|
||||
if(dist.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
|
||||
ret << number
|
||||
ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
|
||||
+5
-5
@@ -393,14 +393,14 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
|
||||
def startDate = null;
|
||||
if(citys.contains(city)){
|
||||
// if(citys.contains(city)){
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
|
||||
}else{
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
}
|
||||
// }else{
|
||||
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// }
|
||||
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
|
||||
+138
-42
@@ -2,15 +2,16 @@ package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
//def citys = CcbCommonUtils.citys;
|
||||
//def rows = []
|
||||
//
|
||||
@@ -25,8 +26,8 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
// println city +" "+count
|
||||
//}
|
||||
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0802(3).xlsx"
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\成交统计.xlsx",0,1)
|
||||
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
|
||||
//def dealMap = [:]
|
||||
//for(def d : datas){
|
||||
// dealMap.put(d['A'], d['B'])
|
||||
@@ -35,52 +36,99 @@ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// d['P'] = dealMap.get(d['C'])
|
||||
// d['Q'] = dealMap.get(d['C'])
|
||||
// }
|
||||
//},2)
|
||||
|
||||
|
||||
def city = "中山"
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
|
||||
def index =0
|
||||
def updateIndex = 0
|
||||
def delIndex = 0
|
||||
for(def dist : dists){
|
||||
if(dist.region=="石岐区街道"){
|
||||
def lat = dist.lat/10000000000.0
|
||||
def lng = dist.lng/10000000000.0
|
||||
def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
|
||||
if(regionStr){
|
||||
def region = RegionData.formatRegion(dist.city, regionStr)
|
||||
def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
|
||||
if(obj){
|
||||
delIndex++
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1)))
|
||||
println delIndex+" [del]"+region +" "+dist.name
|
||||
}else{
|
||||
updateIndex++
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
|
||||
println updateIndex +" [update]"+region+" "+dist.name
|
||||
}
|
||||
}else{
|
||||
index++
|
||||
println index+" [没找到]"+dist.name
|
||||
}
|
||||
|
||||
|
||||
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
|
||||
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
|
||||
//def dealMap = [:]
|
||||
//for(def d : datas){
|
||||
// def map = [:]
|
||||
// map.put("land_area",d['D'])
|
||||
// map.put("build_area",d['F'])
|
||||
// map.put("greeningrate_area",d['H'])
|
||||
// map.put("parkingcnt_area",d['J'])
|
||||
// map.put("zts_area",d['L'])
|
||||
// map.put("buildyear_area",d['N'])
|
||||
// map.put("plotratio",d['P'])
|
||||
//
|
||||
// dealMap.put(d['A'], map)
|
||||
//}
|
||||
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def city = d['B']
|
||||
if(city){
|
||||
def cityid = cityCodeDatas.get(city)
|
||||
def total = findCount(cityid, sqlInstance)
|
||||
d['I'] = total
|
||||
// def a = findEstateType(cityid, sqlInstance)
|
||||
//
|
||||
// d['AA'] = total
|
||||
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
|
||||
//
|
||||
// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
|
||||
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
|
||||
//
|
||||
// def c = findFiledCount(cityid, 'address', sqlInstance)
|
||||
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
|
||||
//
|
||||
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
|
||||
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
|
||||
//
|
||||
// def e = findFiledCount(cityid, 'landarea', sqlInstance)
|
||||
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
|
||||
//
|
||||
// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
|
||||
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
|
||||
//
|
||||
// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
|
||||
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
|
||||
//
|
||||
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
|
||||
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
|
||||
//
|
||||
// def i = findFiledCount(cityid, 'latitude', sqlInstance)
|
||||
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
|
||||
//
|
||||
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
|
||||
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
|
||||
|
||||
println d
|
||||
}
|
||||
},0)
|
||||
|
||||
def findEstateType(cityid, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
def getStreetInfo(def city, def glocation){
|
||||
def township = null
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent && addressComponent.city.contains(city)){
|
||||
township = addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
return township;
|
||||
def findCount(cityid, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
def findFiledCount(cityid, field, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null";
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
@@ -88,6 +136,54 @@ def getStreetInfo(def city, def glocation){
|
||||
|
||||
|
||||
|
||||
//def city = "中山"
|
||||
//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
|
||||
//def index =0
|
||||
//def updateIndex = 0
|
||||
//def delIndex = 0
|
||||
//for(def dist : dists){
|
||||
// if(dist.region=="石岐区街道"){
|
||||
// def lat = dist.lat/10000000000.0
|
||||
// def lng = dist.lng/10000000000.0
|
||||
// def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
|
||||
// if(regionStr){
|
||||
// def region = RegionData.formatRegion(dist.city, regionStr)
|
||||
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
|
||||
// if(obj){
|
||||
// delIndex++
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1)))
|
||||
// println delIndex+" [del]"+region +" "+dist.name
|
||||
// }else{
|
||||
// updateIndex++
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
|
||||
// println updateIndex +" [update]"+region+" "+dist.name
|
||||
// }
|
||||
// }else{
|
||||
// index++
|
||||
// println index+" [没找到]"+dist.name
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//def getStreetInfo(def city, def glocation){
|
||||
// def township = null
|
||||
// if(glocation){
|
||||
// def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
// if(jsonObject && jsonObject.regeocode) {
|
||||
// def addressComponent = jsonObject.regeocode.addressComponent
|
||||
// if(addressComponent && addressComponent.city.contains(city)){
|
||||
// township = addressComponent.township
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return township;
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+2
@@ -780,6 +780,8 @@ class CommonDatautils {
|
||||
keys.put("address", 1);
|
||||
keys.put("build_structure", 1);
|
||||
keys.put("out_pics", 1);
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
|
||||
+38
-13
@@ -29,7 +29,7 @@ import java.util.regex.Pattern;
|
||||
class LocationTest {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -40,9 +40,9 @@ class LocationTest {
|
||||
// System.out.println(str);
|
||||
|
||||
def task = new LocationTest()
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
task.run3()
|
||||
// task.run4()
|
||||
// task.run5()
|
||||
}
|
||||
@@ -162,7 +162,9 @@ class LocationTest {
|
||||
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误-南昌.xlsx", {d->
|
||||
def index = 0
|
||||
ExcelUtils.modify("d:\\数据\\福州\\经纬度错误.xlsx", {d->
|
||||
index++
|
||||
def distance = null
|
||||
if(d['H']){
|
||||
distance = d['H'].toDouble()
|
||||
@@ -261,6 +263,7 @@ class LocationTest {
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
},0)
|
||||
|
||||
}
|
||||
@@ -311,7 +314,7 @@ class LocationTest {
|
||||
}
|
||||
}
|
||||
|
||||
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
def baiduRest = null;//FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
if(baiduRest){
|
||||
bdlocationMap = getMapLocation(dist, baiduRest);
|
||||
if(bdlocationMap){
|
||||
@@ -464,7 +467,11 @@ class LocationTest {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
mapLocation(map)
|
||||
def ret = mapLocation(map)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
println totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -474,14 +481,35 @@ class LocationTest {
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
ExcelUtils.write("d:\\数据\\福州\\经纬度错误.xlsx", rets, "经纬度错误")
|
||||
}
|
||||
|
||||
def mapLocation(map){
|
||||
def dist = CommonDatautils.findDistrictById(map.commid)
|
||||
def dist = CommonDatautils.findDistrictById(map[1])
|
||||
if(dist){
|
||||
|
||||
def loactionMap = matchMap(dist)
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
if (loactionMap) {
|
||||
ret << loactionMap.source
|
||||
ret << loactionMap.region
|
||||
ret << loactionMap.mapName
|
||||
ret << loactionMap.address
|
||||
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
ret << distance
|
||||
ret << loactionMap.lng
|
||||
ret << loactionMap.lat
|
||||
if (dist.region == loactionMap.region) {
|
||||
println dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
|
||||
} else {
|
||||
println dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
@@ -494,10 +522,7 @@ class LocationTest {
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
rets << [r.cityid, r.commid]
|
||||
});
|
||||
|
||||
def map = [:]
|
||||
|
||||
return map
|
||||
return rets
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ class AddressStandard {
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
},0, "Sheet1")
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -60,7 +60,8 @@ public class ExcelUtils{
|
||||
FileInputStream inp = null;
|
||||
try {
|
||||
inp = new FileInputStream(file);
|
||||
Workbook workbook = WorkbookFactory.create(inp)
|
||||
BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 200)
|
||||
Workbook workbook = WorkbookFactory.create(bufferedInputStream)
|
||||
workbook.setForceFormulaRecalculation(true)
|
||||
Sheet sheet = workbook.getSheetAt(sheetIndex);
|
||||
//行号
|
||||
@@ -82,6 +83,9 @@ public class ExcelUtils{
|
||||
}
|
||||
}
|
||||
ret.add(rowData);
|
||||
if (ret.size() % 10000 == 0) {
|
||||
println(ret.size())
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
|
||||
Reference in New Issue
Block a user