批量地址拆分

This commit is contained in:
2022-10-27 16:37:02 +08:00
parent aa25d429d2
commit 22c2852dcd
@@ -0,0 +1,80 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AddressCut extends BaseBatch{
static main(args) {
runBatch(new AddressCut(), args);
}
@Override
public void process(Object args) {
def citys = CommonDatautils.findAllCity()
def index = 0
def fileName = "/home/goojia/gj/address_split.txt"
if (CommonDatautils.isWindow()) {
fileName = "D:\\Êý¾Ý\\address_split.txt"
}
for(def c : citys) {
index++
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"])
def rets = cut(dists)
FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true);
}
}
def cut(rets){
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final CountDownLatch latch = new CountDownLatch(rets.size())
def totalCount = new AtomicLong()
def rows = new CopyOnWriteArrayList()
for(def i=0;i<rets.size();i++){
final def d = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(d.address){
def jsonObj = FangjiaClientUtils.addressStructure(d.city, d.address)
if(jsonObj && jsonObj.result){
def udata = new BasicDBObject()
def street = null
def street_number = null
if(jsonObj.result.street){
street = jsonObj.result.street
}
if(jsonObj.result.streetNumber){
street_number = jsonObj.result.streetNumber
}
if(street || street_number){
rows << [d.city,d.region, d.name, d.address, street, street_number]
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rows;
}
}