批量地址拆分
This commit is contained in:
+80
@@ -0,0 +1,80 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AddressCut extends BaseBatch{
|
||||
static main(args) {
|
||||
runBatch(new AddressCut(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
|
||||
def fileName = "/home/goojia/gj/address_split.txt"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\Êý¾Ý\\address_split.txt"
|
||||
}
|
||||
for(def c : citys) {
|
||||
index++
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"])
|
||||
def rets = cut(dists)
|
||||
FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true);
|
||||
}
|
||||
}
|
||||
|
||||
def cut(rets){
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
def totalCount = new AtomicLong()
|
||||
def rows = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def d = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(d.address){
|
||||
def jsonObj = FangjiaClientUtils.addressStructure(d.city, d.address)
|
||||
if(jsonObj && jsonObj.result){
|
||||
def udata = new BasicDBObject()
|
||||
def street = null
|
||||
def street_number = null
|
||||
if(jsonObj.result.street){
|
||||
street = jsonObj.result.street
|
||||
}
|
||||
if(jsonObj.result.streetNumber){
|
||||
street_number = jsonObj.result.streetNumber
|
||||
}
|
||||
if(street || street_number){
|
||||
rows << [d.city,d.region, d.name, d.address, street, street_number]
|
||||
// udata.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user