update
This commit is contained in:
+6
-2
@@ -85,6 +85,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
|
||||
def startDate = lastMonth+"18"
|
||||
def endDate = curMonth+"19"
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
@@ -196,6 +197,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
@@ -311,7 +313,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.room_structure_code==null?"":m.room_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
@@ -670,6 +672,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
@@ -731,7 +734,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range).must(date_range_mdate)
|
||||
.must(date_range).must(area_range).must(date_range_mdate).must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
@@ -882,6 +885,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
|
||||
+40
@@ -52,6 +52,8 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
exportBigdataBuildingFJW()//
|
||||
exportBigdataCommfeeGj()//
|
||||
|
||||
//exportGuidedPrice()//10月10号再同步数据
|
||||
|
||||
exportBigdataCommbuildstruGJ()//
|
||||
exportBigdataCommhousetypeGJ()//
|
||||
exportBigdataCommbuildyearGj()//
|
||||
@@ -188,6 +190,44 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportGuidedPrice() {
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_fsid
|
||||
data << r.gujia_bds_id
|
||||
data << r.comm_nm
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.gujia_drc_dt
|
||||
data << r.gujia_bds_drc_prc
|
||||
data << r.gujia_add_psn
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_src_co
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
println "GUIDED_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataCommbuildstruGJ(){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
|
||||
+1
-1
@@ -255,7 +255,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","华润佘山九里二期")
|
||||
// q.put("name","文华楼")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
|
||||
+139
-10
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
@@ -38,10 +39,9 @@ class AmapAddressUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new AmapAddressUpdate()
|
||||
def path = "d:\\数据\\小区\\福建省小区地址与高德地图V2.xlsx"
|
||||
// def addr = task.formatAddress("福州", "北三环路201号")
|
||||
// println addr
|
||||
task.run(path)
|
||||
def path = "d:\\数据\\小区\\重庆小区地址与高德地图V1.xlsx"
|
||||
|
||||
// task.run(path)
|
||||
// task.similarMatch(path)
|
||||
// task.addressRun(path)
|
||||
// task.addressSource(path)
|
||||
@@ -50,11 +50,15 @@ class AmapAddressUpdate {
|
||||
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
|
||||
// task.updateData(path)
|
||||
// task.updateLocation(path)//更新经纬度
|
||||
|
||||
task.matchNoSameRegionDist()
|
||||
}
|
||||
def run(path){
|
||||
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"]//CommonDatautils.findAllCity()
|
||||
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"
|
||||
def citys = ["重庆"]//CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
|
||||
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度","得分","匹配类型","小区经纬度与高德距离","操作","来源","地址与小区经纬度距离","备注","经纬度是否修改","高德类型"]
|
||||
|
||||
for (def city : citys) {
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
|
||||
def rets = findAmapDetail(disticts)
|
||||
@@ -63,6 +67,17 @@ class AmapAddressUpdate {
|
||||
ExcelUtils.write(path, allrets, "数据")
|
||||
}
|
||||
|
||||
def matchNoSameRegionDist(){
|
||||
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
|
||||
for (def city : citys) {
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address","estate_type2", "addr_alias","lat", "lng"])
|
||||
def rets = findDiffRegionAmapDetail(disticts)
|
||||
ExcelUtils.write("D:\\数据\\小区\\区域有问题小区.xlsx", rets, city)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def amapRun(){
|
||||
def city = "福州市"
|
||||
def datas = findAllAmapByCity(city)
|
||||
@@ -177,7 +192,15 @@ class AmapAddressUpdate {
|
||||
row << formatAddress(d.city, dbo.address)
|
||||
row << dbo.lng
|
||||
row << dbo.lat
|
||||
row << 1
|
||||
row << dbo.match_type
|
||||
if(d.lng && dbo.lng){
|
||||
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
|
||||
row << distance
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
@@ -186,6 +209,8 @@ class AmapAddressUpdate {
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
}
|
||||
println totalCount.get() +' '+row
|
||||
rets << row
|
||||
@@ -201,6 +226,75 @@ class AmapAddressUpdate {
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findDiffRegionAmapDetail(districts){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
rets << ["城市","区域","名称","地址","物业类型","地图"]
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def dbos = getAmapLocations(d.city+"市", d.name)
|
||||
if(!dbos){
|
||||
def newName = d.name
|
||||
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
|
||||
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
|
||||
}
|
||||
dbos = getAmapLocations(d.city+"市", newName)
|
||||
}
|
||||
|
||||
if(dbos){
|
||||
def sameFlag = false
|
||||
for(def dbo : dbos){
|
||||
def newRegion = RegionData.formatRegion(d.city, dbo.region)
|
||||
if(d.region == newRegion){
|
||||
sameFlag = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if(!sameFlag){
|
||||
def row = []
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
row << d.address
|
||||
row << d.estate_type2
|
||||
def str = ""
|
||||
for(def dbo : dbos){
|
||||
def distance = null
|
||||
if(d.lng && dbo.lng){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
|
||||
}
|
||||
str = str + dbo.region+"-"+dbo.name+"-"+dbo.address+"-"+dbo.type+"-"+distance+"\n"
|
||||
}
|
||||
row << str
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
println totalCount.get()+" "+d.city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
def formatAddress(def city, def address){
|
||||
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
|
||||
def address_str = null
|
||||
@@ -267,6 +361,38 @@ class AmapAddressUpdate {
|
||||
}
|
||||
|
||||
|
||||
def getAmapLocations(city, name){
|
||||
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
sqlBuffer.append(" and type like '%住宅%'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def list = []
|
||||
while (rs.next()) {
|
||||
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
|
||||
def mapName = rs.getString("name").replace("·","")
|
||||
if(mapName == name){
|
||||
def map = [:]
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
map.region = rs.getString("adname")
|
||||
map.type = rs.getString("type")
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
list << map
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -451,9 +577,11 @@ class AmapAddressUpdate {
|
||||
}
|
||||
|
||||
def updateLocation(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { row ->
|
||||
|
||||
if(row['R'] && row['R']=="是"){
|
||||
index++
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
if(dist1){
|
||||
def udata = new BasicDBObject()
|
||||
@@ -514,8 +642,9 @@ class AmapAddressUpdate {
|
||||
row['J'] = dbo.lng
|
||||
row['K'] = dbo.lat
|
||||
row['L'] = dbo.score
|
||||
row['M'] = dbo.distance
|
||||
row['S'] = dbo.type
|
||||
row['N'] = dbo.distance
|
||||
row['M'] = dbo.match_type
|
||||
row['T'] = dbo.type
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -539,7 +668,7 @@ class AmapAddressUpdate {
|
||||
def mapName = rs.getString("name").replace("·","")
|
||||
|
||||
println name +" "+ mapName
|
||||
if(rs.getString("adname").contains(region)){
|
||||
if(rs.getString("adname") && rs.getString("adname").contains(region)){
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
|
||||
|
||||
+36
-2
@@ -230,12 +230,46 @@ class CcbCodeUtils {
|
||||
}
|
||||
return "11"
|
||||
}
|
||||
//室内结构
|
||||
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
|
||||
|
||||
static def getHouseStructureCode(){
|
||||
return null
|
||||
}
|
||||
|
||||
//室内结构
|
||||
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
|
||||
static def getRoomStructureCode(room_structure){
|
||||
if(room_structure){
|
||||
if(room_structure.contains("平层")){
|
||||
return "11"
|
||||
}else if(room_structure.contains("错层")){
|
||||
return "12"
|
||||
}else if(room_structure.contains("复式")){
|
||||
return "13"
|
||||
}else if(room_structure.contains("阁楼")){
|
||||
return "14"
|
||||
}else if(room_structure.contains("团结户")){
|
||||
return "15"
|
||||
}else if(room_structure.contains("老式房")){
|
||||
return "16"
|
||||
}else if(room_structure.contains("地下室")){
|
||||
return "17"
|
||||
}else if(room_structure.contains("带阁楼")){
|
||||
return "21"
|
||||
}else if(room_structure.contains("带车库")){
|
||||
return "22"
|
||||
}else if(room_structure.contains("带车位")){
|
||||
return "23"
|
||||
}else if(room_structure.contains("带地下室")){
|
||||
return "24"
|
||||
}else if(room_structure.contains("带储藏室")){
|
||||
return "25"
|
||||
}else if(room_structure.contains("跃层")){
|
||||
return "18"
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
//租赁方式 整租:0 合租:1 转租:2
|
||||
static def getRentTypeCode(def rentType){
|
||||
if(rentType){
|
||||
|
||||
+16
-8
@@ -35,15 +35,23 @@ class CityStatExport {
|
||||
}
|
||||
|
||||
def addFieldStat(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d ->
|
||||
def cityInfoDto = CommonDatautils.findCityInfo(d['B'])
|
||||
if(cityInfoDto){
|
||||
def detailMap = cityInfoDto.district_detail
|
||||
d['D'] = detailMap.address.toDouble()*100
|
||||
d['E'] = 100
|
||||
d['F'] = cityInfoDto.price_coverage.toDouble()*100
|
||||
def fileName = "D:\\数据\\fjw.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(!d['B']){
|
||||
def city = RegionData.formatCity(d['A'])
|
||||
if(city){
|
||||
def cityInfoDto = CommonDatautils.findCityInfo(city)
|
||||
if(cityInfoDto){
|
||||
d['B'] = cityInfoDto.existing_district_count
|
||||
def detailMap = cityInfoDto.district_detail
|
||||
// d['D'] = detailMap.address.toDouble()*100
|
||||
// d['E'] = 100
|
||||
d['D'] = cityInfoDto.price_coverage.toDouble()*100
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
},1)
|
||||
|
||||
}
|
||||
|
||||
|
||||
+105
-1
@@ -6,12 +6,15 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -57,7 +60,10 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// synchMapPoi();
|
||||
synchCompany()
|
||||
// synchCompany()
|
||||
// poiCatProcess()
|
||||
// addressAndCatProcess()
|
||||
synchNetwork()
|
||||
}
|
||||
|
||||
def synchCompany(){
|
||||
@@ -368,5 +374,103 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
return townMap
|
||||
}
|
||||
|
||||
def poiCatProcess(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\数据.xlsx",{d->
|
||||
def cat = d['B']
|
||||
if(cat && cat.size()>50){
|
||||
println cat
|
||||
def arrs = cat.split(";")
|
||||
def newCat = ""
|
||||
if(arrs.size()>3){
|
||||
newCat = arrs[0]+";"+arrs[1]+";"+arrs[2]
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("cat",newCat)
|
||||
udata.put("type",arrs[0])
|
||||
udata.put("secondary_type",arrs[1])
|
||||
udata.put("small_type",arrs[2])
|
||||
MAPPOI_CONN.update(new BasicDBObject("cat", cat), new BasicDBObject('$set', udata), false, true)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addressAndCatProcess(){
|
||||
def datas = FileUtils.readTxt("C:\\Users\\gaojuan\\Downloads\\BIGDATA_POI_v1.bad")
|
||||
datas.each{d->
|
||||
def arrs = d.split("@\\|@")
|
||||
if(arrs){
|
||||
def udata = new BasicDBObject()
|
||||
// def address = arrs[5]
|
||||
// if(address.size()>=120){
|
||||
// udata.put("address", "")
|
||||
// }
|
||||
def cat = arrs[8]
|
||||
def catArrs = cat.split(";")
|
||||
if(catArrs.size()>3){
|
||||
cat = catArrs[0]+";"+catArrs[1]+";"+catArrs[2]
|
||||
if(cat.size()>=45){
|
||||
println "取前3依然超:"+cat
|
||||
}else{
|
||||
udata.put("type",catArrs[0])
|
||||
udata.put("secondary_type",catArrs[1])
|
||||
udata.put("small_type",catArrs[2])
|
||||
udata.put("cat", cat)
|
||||
}
|
||||
|
||||
}else{
|
||||
println cat
|
||||
}
|
||||
|
||||
if(udata){
|
||||
MAPPOI_CONN.update(new BasicDBObject("_id", arrs[0]), new BasicDBObject('$set', udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def synchNetwork(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx")
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def location = d['I']
|
||||
def locMap = [:]
|
||||
if(location){
|
||||
def arrs = location.split(",")
|
||||
locMap.lon = arrs[0].toString().toDouble()
|
||||
locMap.lat = arrs[1].toString().toDouble()
|
||||
}
|
||||
rets << ["id":new ObjectId().toString(), "name":d['B'], "address":d['C'], "location":locMap]
|
||||
}
|
||||
|
||||
saveEs("xm_network", rets)
|
||||
}
|
||||
|
||||
def saveEs(index,rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient("192.168.0.12")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest(index, "_doc", map.id).source(map, XContentType.JSON));
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+40
-2
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.script.BaseBatch
|
||||
@@ -9,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -27,6 +29,8 @@ class NetworkDistanceBatch extends BaseBatch{
|
||||
|
||||
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
|
||||
|
||||
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new NetworkDistanceBatch(), args)
|
||||
}
|
||||
@@ -34,11 +38,45 @@ class NetworkDistanceBatch extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def networks = ExcelUtils.read("D:\\数据\\厦门\\网点.xlsx", 0, 1)
|
||||
def networks = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx", 0, 1)
|
||||
def citys = ["厦门","三明","宁德","泉州","福州","漳州","南平","龙岩","莆田","重庆"]
|
||||
for(def c : citys){
|
||||
run(c, "district", networks)
|
||||
}
|
||||
// saveMapData()
|
||||
}
|
||||
|
||||
def saveMapData(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\厦门银行鹭通支行.xlsx",{d->
|
||||
if(d['C']){
|
||||
def arrs = d['C'].split(",")
|
||||
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("network_name", d['B'])
|
||||
obj.put("ntk_gd_lng", arrs[0].toString().toDouble())
|
||||
obj.put("ntk_gd_lat", arrs[1].toString().toDouble())
|
||||
obj.put("ntk_bd_lng", lng2)
|
||||
obj.put("ntk_bd_lat", lat2)
|
||||
obj.put("comm_id", d['D'])
|
||||
obj.put("city", d['E'])
|
||||
obj.put("region", d['F'])
|
||||
obj.put("name", d['G'])
|
||||
obj.put("distance", d['I'].toString().toDouble().toInteger())
|
||||
obj.put("time", d['K'].toString().toDouble().toInteger())
|
||||
obj.put("gh_distance", d['J'].toString().toDouble().toInteger())
|
||||
def dist_loc_arr = d['H'].split(",")
|
||||
|
||||
def baidu_dist = BaiDuApiUtils.geoconv(dist_loc_arr[0].toString().toDouble(), dist_loc_arr[1].toString().toDouble(), 3)
|
||||
obj.put("dist_gd_lng", dist_loc_arr[0].toString().toDouble())
|
||||
obj.put("dist_gd_lat", dist_loc_arr[1].toString().toDouble())
|
||||
obj.put("dist_bd_lng", baidu_dist.x)
|
||||
obj.put("dist_bd_lat", baidu_dist.y)
|
||||
|
||||
MAP_NETWORK_COLL.save(obj)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(city, cat, networks){
|
||||
@@ -81,7 +119,7 @@ class NetworkDistanceBatch extends BaseBatch{
|
||||
def lastRest = null
|
||||
def minDistance = Integer.MAX_VALUE
|
||||
for(def row : networks){
|
||||
def locationstr = row['D']
|
||||
def locationstr = row['E']
|
||||
if(locationstr){
|
||||
def arrs = locationstr.split(",")
|
||||
def distance = GeoUtils.calculateDistance(dist.location.lon, dist.location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
|
||||
|
||||
+22
-4
@@ -23,16 +23,19 @@ class SeaweedDataSynch {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
|
||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new SeaweedDataSynch()
|
||||
// def addr = ["a","b",'c',"d","e","f"]
|
||||
// println addr[0..4]
|
||||
task.run()
|
||||
// println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
@@ -45,7 +48,7 @@ class SeaweedDataSynch {
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
|
||||
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
@@ -92,13 +95,28 @@ class SeaweedDataSynch {
|
||||
def createData(dist){
|
||||
def map = [:]
|
||||
map._id = dist._id
|
||||
map.province = provinceMap.get(dist.city)
|
||||
map.city = dist.city
|
||||
map.region = dist.region
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
map.block = dist.block
|
||||
map.alias= dist.alias
|
||||
map.addr_alias = dist.addr_alias
|
||||
if(dist.mapping_alias){
|
||||
if(dist.mapping_alias.size()>5){
|
||||
map.alias= dist.mapping_alias[0..4]
|
||||
}else{
|
||||
map.alias= dist.mapping_alias
|
||||
}
|
||||
}
|
||||
|
||||
if(dist.addr_alias){
|
||||
if(dist.addr_alias.size()>5){
|
||||
map.addr_alias = dist.addr_alias[0..4]
|
||||
}else{
|
||||
map.addr_alias = dist.addr_alias
|
||||
}
|
||||
}
|
||||
|
||||
map.category = dist.cat
|
||||
map.estate_type = dist.estate_type2
|
||||
map.status = dist.status
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 10
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 2
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
final double total = body.size()
|
||||
|
||||
+4
-4
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230801", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230901", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -23,7 +23,7 @@ while (startDate.before(endDate)) {
|
||||
def nextDate = calendar.getTime()
|
||||
count++
|
||||
///property/addressEvaluate /address/financialRating /v2/property/detail
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/address/financialRating"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
// if (!hashSet.contains(result)) {
|
||||
def item = [doc._id.toString(), "ÁÉÄþ׿ÐÅÆÕ»ÝÕ÷ÐÅ", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
@@ -39,6 +39,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情")
|
||||
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202309.xlsx", data, "财力评级")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user