Merge branch 'script_gj' into 'master'

update

See merge request !717
This commit is contained in:
2023-09-06 13:30:00 +08:00
11 changed files with 410 additions and 35 deletions
@@ -85,6 +85,7 @@ class BigdataForsaleBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def startDate = lastMonth+"18"
def endDate = curMonth+"19"
println "房源时间:"+startDate+" "+endDate
@@ -196,6 +197,7 @@ class BigdataForsaleBatch extends BaseBatch{
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
@@ -311,7 +313,7 @@ class BigdataForsaleBatch extends BaseBatch{
data << m.floor==null?"":m.floor
data << m.height==null?"":m.height
data << m.estate_type==null?"":m.estate_type
data << m.house_structure_code==null?"":m.house_structure_code
data << m.room_structure_code==null?"":m.room_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
data << ""
@@ -670,6 +672,7 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
@@ -731,7 +734,7 @@ class BigdataForsaleBatch extends BaseBatch{
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)
.must(date_range).must(area_range).must(date_range_mdate).must(QueryBuilders.termQuery("root", "lianjia.com"))
if(!alias){
alias = []
@@ -882,6 +885,7 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
@@ -52,6 +52,8 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
//exportGuidedPrice()//10月10号再同步数据
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
@@ -188,6 +190,44 @@ class ExportDistrictRongHe extends BaseBatch{
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
}
def exportGuidedPrice() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_bds_id
data << r.comm_nm
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_drc_dt
data << r.gujia_bds_drc_prc
data << r.gujia_add_psn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
println "GUIDED_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "GUIDED_PRICE_GJ 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def allRows = []
for(def city : citys) {
@@ -255,7 +255,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","华润佘山九里二期")
// q.put("name","文华楼")
DBObject keys = new BasicDBObject();
if(fields){
@@ -1,5 +1,6 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
@@ -38,10 +39,9 @@ class AmapAddressUpdate {
static main(args) {
def task = new AmapAddressUpdate()
def path = "d:\\数据\\小区\\福建省小区地址与高德地图V2.xlsx"
// def addr = task.formatAddress("福州", "北三环路201号")
// println addr
task.run(path)
def path = "d:\\数据\\小区\\重庆小区地址与高德地图V1.xlsx"
// task.run(path)
// task.similarMatch(path)
// task.addressRun(path)
// task.addressSource(path)
@@ -50,11 +50,15 @@ class AmapAddressUpdate {
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
// task.updateData(path)
// task.updateLocation(path)//更新经纬度
task.matchNoSameRegionDist()
}
def run(path){
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"]//CommonDatautils.findAllCity()
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"
def citys = ["重庆"]//CommonDatautils.findAllCity()
def allrets = []
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度","得分","匹配类型","小区经纬度与高德距离","操作","来源","地址与小区经纬度距离","备注","经纬度是否修改","高德类型"]
for (def city : citys) {
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
def rets = findAmapDetail(disticts)
@@ -63,6 +67,17 @@ class AmapAddressUpdate {
ExcelUtils.write(path, allrets, "数据")
}
def matchNoSameRegionDist(){
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
for (def city : citys) {
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address","estate_type2", "addr_alias","lat", "lng"])
def rets = findDiffRegionAmapDetail(disticts)
ExcelUtils.write("D:\\数据\\小区\\区域有问题小区.xlsx", rets, city)
}
}
def amapRun(){
def city = "福州市"
def datas = findAllAmapByCity(city)
@@ -177,7 +192,15 @@ class AmapAddressUpdate {
row << formatAddress(d.city, dbo.address)
row << dbo.lng
row << dbo.lat
row << 1
row << dbo.match_type
if(d.lng && dbo.lng){
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
row << distance
}else{
row << ""
}
}else{
row << ""
row << ""
@@ -186,6 +209,8 @@ class AmapAddressUpdate {
row << ""
row << ""
row << ""
row << ""
row << ""
}
println totalCount.get() +' '+row
rets << row
@@ -201,6 +226,75 @@ class AmapAddressUpdate {
return rets
}
def findDiffRegionAmapDetail(districts){
def rets = Collections.synchronizedList(new ArrayList());
rets << ["城市","区域","名称","地址","物业类型","地图"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dbos = getAmapLocations(d.city+"市", d.name)
if(!dbos){
def newName = d.name
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
}
dbos = getAmapLocations(d.city+"市", newName)
}
if(dbos){
def sameFlag = false
for(def dbo : dbos){
def newRegion = RegionData.formatRegion(d.city, dbo.region)
if(d.region == newRegion){
sameFlag = true
continue
}
}
if(!sameFlag){
def row = []
row << d.city
row << d.region
row << d.name
row << d.address
row << d.estate_type2
def str = ""
for(def dbo : dbos){
def distance = null
if(d.lng && dbo.lng){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
}
str = str + dbo.region+"-"+dbo.name+"-"+dbo.address+"-"+dbo.type+"-"+distance+"\n"
}
row << str
rets << row
}
}
println totalCount.get()+" "+d.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def formatAddress(def city, def address){
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
def address_str = null
@@ -267,6 +361,38 @@ class AmapAddressUpdate {
}
def getAmapLocations(city, name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and type like '%住宅%'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def list = []
while (rs.next()) {
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
def mapName = rs.getString("name").replace("·","")
if(mapName == name){
def map = [:]
map.name = rs.getString("name")
map.address = rs.getString("address")
map.formatAddress = AddressUtil.formatAddress(city, map.address)
map.region = rs.getString("adname")
map.type = rs.getString("type")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
list << map
}
}
return list;
}
@@ -451,9 +577,11 @@ class AmapAddressUpdate {
}
def updateLocation(path){
ExcelUtils.multiThreadingModify(path, { row ->
def index = 0
ExcelUtils.modify(path, { row ->
if(row['R'] && row['R']=="是"){
index++
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
if(dist1){
def udata = new BasicDBObject()
@@ -514,8 +642,9 @@ class AmapAddressUpdate {
row['J'] = dbo.lng
row['K'] = dbo.lat
row['L'] = dbo.score
row['M'] = dbo.distance
row['S'] = dbo.type
row['N'] = dbo.distance
row['M'] = dbo.match_type
row['T'] = dbo.type
}
}
}
@@ -539,7 +668,7 @@ class AmapAddressUpdate {
def mapName = rs.getString("name").replace("·","")
println name +" "+ mapName
if(rs.getString("adname").contains(region)){
if(rs.getString("adname") && rs.getString("adname").contains(region)){
map.name = rs.getString("name")
map.address = rs.getString("address")
@@ -230,12 +230,46 @@ class CcbCodeUtils {
}
return "11"
}
//室内结构
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
static def getHouseStructureCode(){
return null
}
//室内结构
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
static def getRoomStructureCode(room_structure){
if(room_structure){
if(room_structure.contains("平层")){
return "11"
}else if(room_structure.contains("错层")){
return "12"
}else if(room_structure.contains("复式")){
return "13"
}else if(room_structure.contains("阁楼")){
return "14"
}else if(room_structure.contains("团结户")){
return "15"
}else if(room_structure.contains("老式房")){
return "16"
}else if(room_structure.contains("地下室")){
return "17"
}else if(room_structure.contains("带阁楼")){
return "21"
}else if(room_structure.contains("带车库")){
return "22"
}else if(room_structure.contains("带车位")){
return "23"
}else if(room_structure.contains("带地下室")){
return "24"
}else if(room_structure.contains("带储藏室")){
return "25"
}else if(room_structure.contains("跃层")){
return "18"
}
}
return null;
}
//租赁方式 整租:0 合租:1 转租:2
static def getRentTypeCode(def rentType){
if(rentType){
@@ -35,15 +35,23 @@ class CityStatExport {
}
def addFieldStat(){
ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d ->
def cityInfoDto = CommonDatautils.findCityInfo(d['B'])
if(cityInfoDto){
def detailMap = cityInfoDto.district_detail
d['D'] = detailMap.address.toDouble()*100
d['E'] = 100
d['F'] = cityInfoDto.price_coverage.toDouble()*100
def fileName = "D:\\数据\\fjw.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
if(!d['B']){
def city = RegionData.formatCity(d['A'])
if(city){
def cityInfoDto = CommonDatautils.findCityInfo(city)
if(cityInfoDto){
d['B'] = cityInfoDto.existing_district_count
def detailMap = cityInfoDto.district_detail
// d['D'] = detailMap.address.toDouble()*100
// d['E'] = 100
d['D'] = cityInfoDto.price_coverage.toDouble()*100
}
}
}
})
},1)
}
@@ -6,12 +6,15 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -57,7 +60,10 @@ class GaoDePoiDataProcess extends BaseBatch{
@Override
void process(Object args) {
// synchMapPoi();
synchCompany()
// synchCompany()
// poiCatProcess()
// addressAndCatProcess()
synchNetwork()
}
def synchCompany(){
@@ -368,5 +374,103 @@ class GaoDePoiDataProcess extends BaseBatch{
return townMap
}
def poiCatProcess(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\数据.xlsx",{d->
def cat = d['B']
if(cat && cat.size()>50){
println cat
def arrs = cat.split(";")
def newCat = ""
if(arrs.size()>3){
newCat = arrs[0]+";"+arrs[1]+";"+arrs[2]
def udata = new BasicDBObject()
udata.put("cat",newCat)
udata.put("type",arrs[0])
udata.put("secondary_type",arrs[1])
udata.put("small_type",arrs[2])
MAPPOI_CONN.update(new BasicDBObject("cat", cat), new BasicDBObject('$set', udata), false, true)
}
}
})
}
def addressAndCatProcess(){
def datas = FileUtils.readTxt("C:\\Users\\gaojuan\\Downloads\\BIGDATA_POI_v1.bad")
datas.each{d->
def arrs = d.split("@\\|@")
if(arrs){
def udata = new BasicDBObject()
// def address = arrs[5]
// if(address.size()>=120){
// udata.put("address", "")
// }
def cat = arrs[8]
def catArrs = cat.split(";")
if(catArrs.size()>3){
cat = catArrs[0]+";"+catArrs[1]+";"+catArrs[2]
if(cat.size()>=45){
println "取前3依然超:"+cat
}else{
udata.put("type",catArrs[0])
udata.put("secondary_type",catArrs[1])
udata.put("small_type",catArrs[2])
udata.put("cat", cat)
}
}else{
println cat
}
if(udata){
MAPPOI_CONN.update(new BasicDBObject("_id", arrs[0]), new BasicDBObject('$set', udata), false, true)
}
}
}
}
def synchNetwork(){
def datas = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx")
def rets = []
for(def d : datas){
def location = d['I']
def locMap = [:]
if(location){
def arrs = location.split(",")
locMap.lon = arrs[0].toString().toDouble()
locMap.lat = arrs[1].toString().toDouble()
}
rets << ["id":new ObjectId().toString(), "name":d['B'], "address":d['C'], "location":locMap]
}
saveEs("xm_network", rets)
}
def saveEs(index,rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient("192.168.0.12")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest(index, "_doc", map.id).source(map, XContentType.JSON));
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeoUtils
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.BaseBatch
@@ -9,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -27,6 +29,8 @@ class NetworkDistanceBatch extends BaseBatch{
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "192.168.0.43")
static main(args) {
runBatch(new NetworkDistanceBatch(), args)
}
@@ -34,11 +38,45 @@ class NetworkDistanceBatch extends BaseBatch{
@Override
void process(Object args) {
def networks = ExcelUtils.read("D:\\数据\\厦门\\网点.xlsx", 0, 1)
def networks = ExcelUtils.read("D:\\数据\\厦门\\网点经纬度-人工核实(1).xlsx", 0, 1)
def citys = ["ÏÃÃÅ","ÈýÃ÷","ÄþµÂ","ȪÖÝ","¸£ÖÝ","ÕÄÖÝ","ÄÏÆ½","ÁúÑÒ","ÆÎÌï","ÖØÇì"]
for(def c : citys){
run(c, "district", networks)
}
// saveMapData()
}
def saveMapData(){
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\厦门银行鹭通支行.xlsx",{d->
if(d['C']){
def arrs = d['C'].split(",")
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
def obj = new BasicDBObject()
obj.put("network_name", d['B'])
obj.put("ntk_gd_lng", arrs[0].toString().toDouble())
obj.put("ntk_gd_lat", arrs[1].toString().toDouble())
obj.put("ntk_bd_lng", lng2)
obj.put("ntk_bd_lat", lat2)
obj.put("comm_id", d['D'])
obj.put("city", d['E'])
obj.put("region", d['F'])
obj.put("name", d['G'])
obj.put("distance", d['I'].toString().toDouble().toInteger())
obj.put("time", d['K'].toString().toDouble().toInteger())
obj.put("gh_distance", d['J'].toString().toDouble().toInteger())
def dist_loc_arr = d['H'].split(",")
def baidu_dist = BaiDuApiUtils.geoconv(dist_loc_arr[0].toString().toDouble(), dist_loc_arr[1].toString().toDouble(), 3)
obj.put("dist_gd_lng", dist_loc_arr[0].toString().toDouble())
obj.put("dist_gd_lat", dist_loc_arr[1].toString().toDouble())
obj.put("dist_bd_lng", baidu_dist.x)
obj.put("dist_bd_lat", baidu_dist.y)
MAP_NETWORK_COLL.save(obj)
}
})
}
def run(city, cat, networks){
@@ -81,7 +119,7 @@ class NetworkDistanceBatch extends BaseBatch{
def lastRest = null
def minDistance = Integer.MAX_VALUE
for(def row : networks){
def locationstr = row['D']
def locationstr = row['E']
if(locationstr){
def arrs = locationstr.split(",")
def distance = GeoUtils.calculateDistance(dist.location.lon, dist.location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
@@ -23,16 +23,19 @@ class SeaweedDataSynch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
public static def provinceMap = CommonDatautils.findAllCityProvince()
static main(args) {
def task = new SeaweedDataSynch()
// def addr = ["a","b",'c',"d","e","f"]
// println addr[0..4]
task.run()
// println new ObjectId("622c433afb459ab08fdab460").getDate()
}
def run(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
@@ -45,7 +48,7 @@ class SeaweedDataSynch {
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -92,13 +95,28 @@ class SeaweedDataSynch {
def createData(dist){
def map = [:]
map._id = dist._id
map.province = provinceMap.get(dist.city)
map.city = dist.city
map.region = dist.region
map.name = dist.name
map.address = dist.address
map.block = dist.block
map.alias= dist.alias
map.addr_alias = dist.addr_alias
if(dist.mapping_alias){
if(dist.mapping_alias.size()>5){
map.alias= dist.mapping_alias[0..4]
}else{
map.alias= dist.mapping_alias
}
}
if(dist.addr_alias){
if(dist.addr_alias.size()>5){
map.addr_alias = dist.addr_alias[0..4]
}else{
map.addr_alias = dist.addr_alias
}
}
map.category = dist.cat
map.estate_type = dist.estate_type2
map.status = dist.status
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 10
def nThread = Runtime.getRuntime().availableProcessors() * 2
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
final double total = body.size()
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230801", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230901", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -23,7 +23,7 @@ while (startDate.before(endDate)) {
def nextDate = calendar.getTime()
count++
///property/addressEvaluate /address/financialRating /v2/property/detail
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/address/financialRating"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
// if (!hashSet.contains(result)) {
def item = [doc._id.toString(), "ÁÉÄþ׿ÐÅÆÕ»ÝÕ÷ÐÅ", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
@@ -39,6 +39,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情")
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202309.xlsx", data, "财力评级")