小区master分支开发合并

This commit is contained in:
sjshenjian
2021-05-17 20:58:57 +08:00
parent 034f924f50
commit 113f2e8c50
17 changed files with 1613 additions and 0 deletions
@@ -0,0 +1,71 @@
package com.fangjia.script.temp.shenjian.company
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils;
/**
*
* ÐÂÔö¹«Ë¾
*
* @author shenjian
* @since 2021-03-16
*/
class AddCompany {
static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136")
static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136")
static final Set developCompanySet = new HashSet()
static final Set propertyCompanySet = new HashSet()
static final Set actualDevelopCompanySet = new HashSet()
static final Set actualPropertyCompanySet = new HashSet()
static {
develop_company.find().batchSize(10000).each { obj ->
developCompanySet.add(obj.company)
}
property_company.find().batchSize(10000).each { obj ->
propertyCompanySet.add(obj.company)
}
}
static void main(String[] args) {
DBObject query = new BasicDBObject()
query.append("cat", "district").append("visible", 0).append("city", "¹þ¶û±õ")
DbUtils.seaweed.find(query,
new BasicDBObject("developer", 1).append("estate_company", 1))
.batchSize(10000).each {obj ->
actualDevelopCompanySet.add(obj.developer)
actualPropertyCompanySet.add(obj.estate_company)
}
def developCompanySaves = []
def propertyCompanySaves = []
for (String name : actualDevelopCompanySet) {
if (StringUtils.isBlank(name) || developCompanySet.contains(name)) {
continue;
}
def item = new BasicDBObject()
item.company = name
developCompanySaves << item
}
for (String name : actualPropertyCompanySet) {
if (StringUtils.isBlank(name) || propertyCompanySet.contains(name)) {
continue;
}
def item = new BasicDBObject()
item.company = name
propertyCompanySaves << item
}
develop_company.insert(developCompanySaves)
property_company.insert(propertyCompanySaves)
}
}
@@ -0,0 +1,33 @@
package com.fangjia.script.temp.shenjian.company
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang3.StringUtils;
/**
*
* ¿ª·¢É̹«Ë¾
*
* @author shenjian
* @since 2021-03-13
*/
class DevelopCompany {
static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136")
static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136")
static void main(String[] args) {
def companySaves = []
List result = seaweed.distinct("developer")
for (String name : result) {
if (StringUtils.isBlank(name)) {
continue;
}
def item = new BasicDBObject()
item.company = name
companySaves << item
}
develop_company.insert(companySaves)
}
}
@@ -0,0 +1,34 @@
package com.fangjia.script.temp.shenjian.company
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang3.StringUtils
/**
* ÎïÒµ¹«Ë¾
*
* @author shenjian
* @since 2021-03-13
*/
class PropertyCompany {
static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136")
static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136")
static void main(String[] args) {
// ÎïÒµ¹«Ë¾
def companySaves = []
List result = seaweed.distinct("estate_company")
for (String name : result) {
if (StringUtils.isBlank(name)) {
continue;
}
def item = new BasicDBObject()
item.company = name
companySaves << item
}
property_company.insert(companySaves)
}
}
@@ -0,0 +1,16 @@
package com.fangjia.script.temp.shenjian.dto
/**
* Æ¥Åä·µ»Ø½á¹¹Ìå
*/
class SearchAddressDTO {
String code;
String msg;
Result result;
class Result {
String street
String streetNumber
}
}
@@ -0,0 +1,116 @@
package com.fangjia.script.temp.shenjian.fill
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.temp.shenjian.util.DistrictUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.GeoHelper
import groovy.util.logging.Slf4j
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 根据周围500米容积率与绿化率平均值推算小区
*
* @author shenjian*
* @since 2021-03-13
*/
@Slf4j
class CalcCubageAndVirescence {
static main(args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("CalcCubageAndVirescence")
log.info("开始根据周围500米容积率与绿化率平均值推算小区")
def cities = ['北京', '重庆', '哈尔滨']
for (def city : cities) {
def sameRegionDistricts = [:]
def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts);
def result = []
districts.each { obj ->
if (obj.finish_building_date) {
int year = CommonUtils.getBuildYear(obj.finish_building_date)
if (year > 2000) {
if (!(obj.cubage_percent && obj.virescence_percent)) {
def temp = this.findResult(obj.lat2 / 10000000000.0d, obj.lng2 / 10000000000.0d, 500, sameRegionDistricts.get(obj.region), obj.name, obj._id)
result << temp
}
}
}
}
def headers = ["小区ID", "容积率", "绿化率"]
FileUtils.batchCreateFile(result, "d:\\数据\\小区\\补充容积绿化" + city + ".csv", headers, "UTF-8")
}
stopWatch.stop()
log.info("完成根据周围500米容积率与绿化率平均值推算, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
}
static def findResult(lat, lng, radius, districts, region, id) {
def avgMap = [:]
def ret = [id]
if (lat && lng) {
def cubagePercentArr = []
def virescencePercentArr = []
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, radius)
for (def row : districts) {
if (row.lat2 && row.lng2) {
def lat2 = row.lat2 / 10000000000.0;
def lng2 = row.lng2 / 10000000000.0;
if (lat2 >= map.minLat && lat2 <= map.maxLat && lng2 >= map.minLng && lng2 <= map.maxLng) {
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
if (distance <= radius) {
if (row.cubage_percent) {
cubagePercentArr << row.cubage_percent;
}
if (row.virescence_percent) {
virescencePercentArr << row.virescence_percent
}
}
}
}
}
try {
// 计算平均容积率
if (cubagePercentArr) {
double sum = 0
double count = 0.0d
for (def m : cubagePercentArr) {
sum = sum + Double.valueOf(m)
count++
}
if (sum) {
double avgCubagePercent = sum / count
ret << DataHandleUtils.getDoubleRound(2, avgCubagePercent)
}
} else {
ret << ""
}
// 计算平均绿化率
if (virescencePercentArr) {
double sum = 0
double count = 0.0d
for (def m : virescencePercentArr) {
sum += Double.valueOf(m)
count++
}
if (sum) {
def avgVirescencePercent = sum / (count)
ret << DataHandleUtils.getDoubleRound(2, avgVirescencePercent)
}
} else {
ret << ""
}
log.info("找到可以优化的小区" + region + "周围平均值为:容积率" + ret[1] + ", 绿化率" + ret[2])
} catch(Exception ex) {
log.error("容积率或绿化率不标准,无法解析")
}
}
return ret
}
}
@@ -0,0 +1,112 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区建筑结构
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class BuildStructure {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static Map buildStructureMap = new HashMap()
static {
buildStructureMap.put("钢结构", "3")
buildStructureMap.put("钢、钢筋混凝土结构", "1")
buildStructureMap.put("钢筋混凝土结构", "1")
buildStructureMap.put("钢混", "1")
buildStructureMap.put("钢混结构", "1")
buildStructureMap.put("混合结构", "11")
buildStructureMap.put("钢筋混凝土", "11")
buildStructureMap.put("砖木结构", "10")
buildStructureMap.put("其他结构", "11")
buildStructureMap.put("砖混结构", "2")
buildStructureMap.put("框架结构", "4")
buildStructureMap.put("框架剪力墙结构", "5")
buildStructureMap.put("剪力墙结构", "6")
buildStructureMap.put("筒体结构", "7")
buildStructureMap.put("框架筒体结构", "8")
buildStructureMap.put("筒中筒", "9")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("BuildStructure")
log.info("开始导出小区建筑结构表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("build_structure", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","建筑结构","备注", "创建时间", "更新时间", "来源公司"]
def seaweedCount = [21596, 17053, 5410];
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger count = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, count)
println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\建筑结构\\bigdata_commbuildstru_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区建筑结构表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << this.getBuildStructure(dbo.get("build_structure")) // 建筑结构
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
if (StringUtils.isNotBlank(dbo.get("build_structure"))) {
rows << row
count.incrementAndGet()
}
})
return rows
};
static def getBuildStructure(def houseStructure) {
def ret
if (StringUtils.isNotBlank(houseStructure)) {
ret = buildStructureMap.get(houseStructure)
if (StringUtils.isBlank(ret)) {
ret = "11" // 其他
}
} else {
ret = ""
}
return ret
}
}
@@ -0,0 +1,129 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区建筑类型表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class BuildType {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static Map buildTypeMap = new HashMap()
static {
buildTypeMap.put("多层", "0104")
buildTypeMap.put("小高层", "0103")
buildTypeMap.put("高层", "0199")
buildTypeMap.put("超高层", "0199")
buildTypeMap.put("花园洋房", "0199")
buildTypeMap.put("老洋房", "0199")
buildTypeMap.put("四合院", "0199")
buildTypeMap.put("独栋", "0105")
buildTypeMap.put("双拼", "0199")
buildTypeMap.put("联排", "0106")
buildTypeMap.put("叠加", "0107")
buildTypeMap.put("老式别墅洋房", "0199")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("BuildStructure")
log.info("开始导出小区建筑类型表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","建筑类型","备注", "创建时间", "更新时间", "来源公司"];
def seaweedCount = [21596, 17053, 5410];
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger count = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, count)
println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\建筑类型\\bigdata_commbuildtype_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区建筑类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
if (StringUtils.isNotBlank(dbo.get("build_type"))) {
Set set = this.getBuildTypeCode(dbo.get("build_type"))
for (String type : set) {
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << type // 建筑类型
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
rows << row
}
if (set.size() > 0) {
count.incrementAndGet()
}
}
})
return rows
}
static Set getBuildTypeCode(def buildType) {
Set<String> set = new HashSet<>()
if (StringUtils.isNotBlank(buildType)) {
String[] types = buildType.split(",")
for (String type : types) {
set.add(buildTypeMap.get(type))
}
if (set.isEmpty()) {
set.add("0199") // 其他
}
}
return set
}
static List getBuildTypeCodeSeq(def buildType) {
List list = new ArrayList()
if (StringUtils.isNotBlank(buildType)) {
String[] types = buildType.split(",")
for (String type : types) {
list.add(buildTypeMap.get(type))
}
if (list.isEmpty()) {
list.add("0199") // 其他
}
}
return list
}
}
@@ -0,0 +1,91 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.text.SimpleDateFormat
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区建筑年代表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class BuildYear {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("BuildYear")
log.info("开始导出小区建筑年代表")
def cites = ["北京", "重庆", "哈尔滨"]
def seaweedCount = [21596, 17053, 5410];
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("finish_building_date", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","建筑年代","备注", "创建时间", "更新时间", "来源公司"]
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger count = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, count)
println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\建筑年代\\bigdata_commbuildyear_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区建筑年代表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
Integer year = CommonUtils.getBuildYear(dbo.get("finish_building_date")) // 建筑年代
if (year > 1900) {
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << year
row << "" // 备注
Integer createYear = CommonUtils.getBuildYear(DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy"))
if (year > createYear) {
row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date());
row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date())
} else {
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
}
row << "fjw" // 来源公司
rows << row
count.incrementAndGet()
}
})
return rows
}
}
@@ -0,0 +1,87 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.core.util.ExcelUtil
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
/**
* @author shenjian
* @since 2021-03-13
*/
class CollectTest {
static void main(String[] args) {
def cites = ["北京", "重庆", "哈尔滨"];
for (def city : cites) {
/**
* _id 小区ID
* name 小区名
* alias 小区曾用名
*/
def districts = this.findAllCityDistricts(city, "district",
["city", "_id", "region", "name", "alias"])
print(districts)
}
}
static def findAllCityDistricts(def city, def cat, def fields) {
DBObject q = new BasicDBObject();
q.put("visible", 0)
q.put("cat", cat)
q.put("region", "密云")
if (city) {
q.put("city", city)
}
DBObject keys = new BasicDBObject();
if (fields) {
for (def field : fields) {
keys.put(field, 1)
}
}
DSCursor cursor = CommonDatautils.getDataStoreConnection().find(q, keys).batchSize(4000)
def districtKeys = []
def rows = [];
rows << ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积",
"小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"]
for (def dbo : cursor) {
def row = [];
row << null // TODO
row << dbo.get("_id")
row << dbo.get("name")
row << dbo.get("alias")
row << null // TODO
row << dbo.get("street_community") // 街道 TODO 拆分为街道门牌号
row << dbo.get("street_community") // 门牌号
rows << dbo.get("address")
rows << dbo.get("all_area")
rows << dbo.get("build_area")
rows << dbo.get("land_property")
ExcelUtils.write("d:\\数据\\小区表.xlsx", rows, "小区")
StringBuilder data = new StringBuilder()
Set<String> keySet = dbo.keySet()
keySet.forEach({key ->
if ("alias" == key) {
def value = dbo.get(key)
if (value != null) {
value = value[0]
}
data.append(value)
} else {
data.append(dbo.get(key))
}
data.append("|@|")
})
districtKeys << StringUtils.substringBeforeLast(data.toString(), "|@|")
};
return districtKeys;
}
}
@@ -0,0 +1,155 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang3.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区公司表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class Company {
private static ExecutorService executorService = Executors.newFixedThreadPool(2);
// 物业公司
static final DBCollection PROPERTY_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136")
// 开发商
static final DBCollection DEVELOP_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136")
static final propertyCompany = new HashMap()
static final developCompany = new HashMap()
static {
PROPERTY_COMPANY_COLLECTION.find().forEach({company ->
propertyCompany.put(company.get("company"), company.get("_id").toString())
})
DEVELOP_COMPANY_COLLECTION.find().forEach({company ->
developCompany.put(company.get("company"), company.get("_id").toString())
})
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("Company")
log.info("开始导出小区公司表")
def cites = ["北京", "重庆", "哈尔滨"]
def seaweedCount = [21596, 17053, 5410];
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("estate_company", 1).append("developer", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","公司ID","公司类型", "备注", "创建时间", "更新时间", "来源公司"]
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger developCount = new AtomicInteger(0)
AtomicInteger estateCount = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, developCount, estateCount)
println("城市" + city + " " + cityId + "物业数量为" + estateCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((estateCount.get() * 100) / seaweedCount[i])) )
println("城市" + city + " " + cityId + "开发商数量为" + developCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((developCount.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\公司\\bigdata_commcomp_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区公司表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger developCount, AtomicInteger estateCount) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
def rowCanSave = true
def row1CanSave = true
row << cityId // 城市ID
row << dbo.get("_id").toString()
def propertyCompany = dbo.get("estate_company")
if (StringUtils.isNotBlank(propertyCompany)) {
row << this.getPropertyCompanyId(propertyCompany) // 公司ID
row << "SV" // 公司类型(物业管理公司SV、 房地产开发商DV )
} else {
rowCanSave = false
}
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
def row1 = []
row1 << cityId
row1 << dbo.get("_id").toString()
def developerCompany = dbo.get("developer")
if (StringUtils.isNotBlank(developerCompany)) {
row1 << this.getDevelopCompanyId(developerCompany) // 公司ID
row1 << "DV" // 公司类型(物业管理公司SV、 房地产开发商DV )
} else {
row1CanSave = false
}
row1 << "" // 备注
row1 << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row1 << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row1 << "fjw" // 来源公司
if (rowCanSave) {
rows << row
estateCount.incrementAndGet()
}
if (row1CanSave) {
rows << row1
developCount.incrementAndGet()
}
})
return rows
}
/**
* 获取开发商公司ID
* @param company 公司名称
* @return
*/
static def getDevelopCompanyId(String company) {
if (StringUtils.isBlank(company)) {
return ""
}
return developCompany.get(company)
}
/**
* 获取物业公司ID
* @param company 公司名称
* @return
*/
static def getPropertyCompanyId(String company) {
if (StringUtils.isBlank(company)) {
return ""
}
return propertyCompany.get(company)
}
}
@@ -0,0 +1,112 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区费用表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class Fee {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("Fee")
log.info("开始导出小区费用表")
def cites = ["北京", "重庆", "哈尔滨"]
def seaweedCount = [21596, 17053, 5410];
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("estate_company", 1).append("estate_charge", 1)
.append("estate_type2", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID", "公司ID", "费用", "房屋用途","建筑类型", "小区费用类型", "费用说明", "备注", "创建时间", "更新时间", "来源公司"]
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger count = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, count)
println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\费用\\bigdata_commfee_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区费用表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def estate_charge = dbo.get("estate_charge")
if (estate_charge && estate_charge.toString().indexOf("~") != -1) {
String[] arr = estate_charge.toString().split("~")
estate_charge = ((Double.valueOf(arr[0]) + Double.valueOf(arr[1])) / 2.0D).toString()
}
if (estate_charge == '-' || estate_charge == '无') {
estate_charge = ""
}
if (StringUtils.isNotBlank(dbo.get("estate_company")) && StringUtils.isNotBlank(estate_charge)
&& StringUtils.isNotBlank(dbo.get("estate_type2")) && StringUtils.isNotBlank(dbo.get("build_type"))) {
List buildTypes = BuildType.getBuildTypeCodeSeq(dbo.get("build_type"))
Set<String> savedBuildTypeSet = new HashSet<>()
boolean isFirst = true
for (String buildType : buildTypes) {
if (savedBuildTypeSet.contains(buildType)) {
continue
}
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString() // 小区ID
row << Company.getPropertyCompanyId(dbo.get("estate_company")) // 公司ID
// 处理 db.getCollection('seaweed').find({'estate_charge': '3.3~4.3'}, {'estate_charge': 1})
// 特殊费用 3.3~4.3
if (isFirst) {
row << estate_charge // 费用
isFirst = false
count.incrementAndGet()
} else {
row << ""
break
}
row << HousePurpose.getHousePurposeCode(dbo.get("estate_type2")) // 房屋用途
row << buildType // 建筑类型
row << "1" // 小区费用类型 物业费1 电梯费2 垃圾费3
row << dbo.get("") // 费用说明
row << dbo.get("") // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
rows << row
savedBuildTypeSet.add(buildType)
}
}
})
return rows
}
}
@@ -0,0 +1,109 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.core.util.StringUtil
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区房屋用途表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class HousePurpose {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static Map housePurposeMap = new HashMap()
static {
housePurposeMap.put("普通住宅", "11")
housePurposeMap.put("别墅", "11")
housePurposeMap.put("商住", "13")
housePurposeMap.put("其他", "99")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("BuildStructure")
log.info("开始导出小区房屋用途表表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("estate_type2", 1).append("household_count", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","房屋用途","户数", "备注", "创建时间", "更新时间", "来源公司"];
for (def city : cites) {
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
def result = this.findAllCityDistricts(city, "district", cityId, filter)
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\房屋用途\\bigdata_commhouseuse_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区房屋用途表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << getHousePurposeCode(dbo.get("estate_type2"))
if (dbo.get("household_count") != null) {
if (Long.valueOf(dbo.get("household_count")) < 50) {
row << "" // 户数
} else {
row << dbo.get("household_count")
}
} else {
row << "" // 户数
}
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
if (dbo.get("m_date")) {
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
} else {
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss")
}
row << "fjw" // 来源公司
if (StringUtils.isNotBlank(dbo.get("estate_type2"))) {
rows << row
}
})
return rows
};
static def getHousePurposeCode(def housePurpose) {
def ret
if (StringUtils.isNotBlank(housePurpose)) {
ret = housePurposeMap.get(housePurpose)
if (StringUtils.isBlank(ret)) {
ret = "99" // 其他
}
} else {
ret = ""
}
return ret
}
}
@@ -0,0 +1,99 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区房屋类型表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class HouseType {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static Map houseTypeMap = new HashMap()
static {
houseTypeMap.put("普通住宅", "01")
houseTypeMap.put("商住", "07")
houseTypeMap.put("别墅", "08")
houseTypeMap.put("其他", "99")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("HouseType")
log.info("开始导出小区房屋类型表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("estate_type2", 1).append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","小区房屋类型","备注", "创建时间", "更新时间", "来源公司"]
for (def city : cites) {
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
def result = this.findAllCityDistricts(city, "district", cityId, filter)
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\房屋类型\\bigdata_commhousetype_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区房屋类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << this.getHouseTypeCode(dbo.get("estate_type2")) // 小区房屋类型
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
if (dbo.get("m_date")) {
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
} else {
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss")
}
row << "fjw" // 来源公司
if (StringUtils.isNotBlank(dbo.get("estate_type2"))) {
rows << row
}
})
return rows
};
static def getHouseTypeCode(def houseType) {
def ret
if (StringUtils.isNotBlank(houseType)) {
ret = houseTypeMap.get(houseType) // 房屋类型
if (StringUtils.isBlank(ret)) {
ret = "99" // 其他
}
} else {
ret = ""
}
return ret
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区地理位置表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class Position {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("Location")
log.info("开始导出小区地理位置表结构表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("lng", 1).append("lat", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","物体类型","物体ID","位置类型", "序号", "经度", "纬度", "海拔", "备注", "创建时间", "更新时间", "来源公司"]
for (def city : cites) {
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
def result = this.findAllCityDistricts(city, "district", cityId, filter)
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\地理位置\\bigdata_position_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区地理位置表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
row << cityId // 城市ID
row << "5" // 物体类型 小区5
row << dbo.get("_id").toString() // 物体ID
row << "1" // 位置类型 中心点1
row << "1" // 序号
row << DataHandleUtils.getDoubleRound(6, dbo.get("lng") / 10000000000d) // 经度
row << DataHandleUtils.getDoubleRound(6, dbo.get("lat") / 10000000000d) // 纬度
row << "" // 海拔
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
rows << row
})
return rows
}
}
@@ -0,0 +1,288 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.temp.shenjian.dto.SearchAddressDTO
import com.fangjia.script.temp.shenjian.fill.CalcCubageAndVirescence
import com.fangjia.script.temp.shenjian.util.DistrictUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.regex.Pattern
/**
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class Seaweed {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
private static HttpRequest http = new HttpRequest()
private static final String TOKEN = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
private static Map seaweedTypeMap = new HashMap();
private static Pattern pattern = Pattern.compile("\\s*|\t|\r|\n"); // 取消空格换行等
private static Map cityDistrictMap = new HashMap();
static {
seaweedTypeMap.put("普通住宅", "pa101")
seaweedTypeMap.put("别墅", "pa101")
seaweedTypeMap.put("商住", "pa101")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("Seaweed")
log.info("开始导出小区公司表")
//def cites = ["北京", "重庆", "哈尔滨"]
//def seaweedCount = [21596, 17053, 5410];
def cites = ["哈尔滨"]
def seaweedCount = [5410];
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("name", 1).append("alias", 1).append("street_community", 1)
.append("address", 1).append("all_area", 1).append("build_area", 1).append("region", 1)
.append("estate_type2", 1).append("virescence_percent", 1).append("cubage_percent", 1)
.append("car_site_count", 1).append("c_date", 1).append("m_date", 1)
.append("lng", 1).append("lat", 1)
def headers = ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积",
"小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"]
for (int i = 0; i < cites.size(); i++) {
final t = i;
def city = cites[t]
def sameRegionDistricts = [:]
def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts)
BasicDBObject cityDistrictInfo = FangJiaAreaMapUtils.convertToOfficial(city)
def cityId = cityDistrictInfo.get("city_code") // 城市ID
final def map = this.findAllCityDistricts(city, "district", cityId, filter, sameRegionDistricts)
executorService.execute(new Runnable() {
@Override
void run() {
def result = []
AtomicInteger virescenceCount = new AtomicInteger(0) // 绿化率
AtomicInteger cubageCount = new AtomicInteger(0) // 容积率
AtomicInteger allAreaCount = new AtomicInteger(0)
AtomicInteger buildAreaCount = new AtomicInteger(0)
for (def key : map.keys()) {
result << map.get(key)
if (map.get(key) && map.get(key)[8] != "") {
allAreaCount.incrementAndGet()
}
if (map.get(key) && map.get(key)[9] != "") {
buildAreaCount.incrementAndGet()
}
if (map.get(key) && map.get(key)[12] != "") {
virescenceCount.incrementAndGet()
}
if (map.get(key) && map.get(key)[13] != "") {
cubageCount.incrementAndGet()
}
}
println("城市" + city + " " + cityId + "绿化率数量为" + virescenceCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((virescenceCount.get() * 100) / seaweedCount[t])) )
println("城市" + city + " " + cityId + "容积率数量为" + cubageCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((cubageCount.get() * 100) / seaweedCount[t])) )
println("城市" + city + " " + cityId + "占地面积数量为" + allAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((allAreaCount.get() * 100) / seaweedCount[t])) )
println("城市" + city + " " + cityId + "建筑面积数量为" + buildAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((buildAreaCount.get() * 100) / seaweedCount[t])) )
FileUtils.batchCreateFile(result, "d:\\数据\\小区\\bigdata_community_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
fixedThreadPool.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, def sameRegionDistricts) {
def map = new ConcurrentHashMap();
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
def total = DbUtils.seaweed.find(query, filter).count()
final CountDownLatch latch = new CountDownLatch(total)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
fixedThreadPool.execute(new Runnable() {
@Override
void run() {
try {
dealSpecialChar(dbo)
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << dbo.get("name")
if (dbo.get("alias") != null && dbo.get("alias")[0] != null) {
row << pattern.matcher(dbo.get("alias")[0]).replaceAll("")
} else {
row << ""
}
def districtId
if (StringUtils.isBlank(cityDistrictMap.get(dbo.get("region")))) {
districtId = FangJiaAreaMapUtils.convertToOfficial(city, dbo.get("region")).get("district_code")
// 行政区ID
cityDistrictMap.put(dbo.get("region"), districtId)
} else {
districtId = cityDistrictMap.get(dbo.get("region"))
}
row << districtId
Map params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city);
params.put("address", dbo.get("address"));
// 暂时不用解析
// String s = http.get("http://open.fangjia.com/address/standardize", params, "utf-8");
// SearchAddressDTO searchAddress = com.alibaba.fastjson.JSONObject.parseObject(s, SearchAddressDTO.class)
// if (searchAddress.code == '200') {
// //row << searchAddress.result.street // 街道
// row << ""
// row << searchAddress.result.streetNumber // 门牌号
// } else {
// row << ""
// row << ""
// }
row << "" // 街道
row << "" // 门牌号
row << dbo.get("address")
if (dbo.get("all_area") && dbo.get("all_area").toString().indexOf("公顷") != -1) {
def val = dbo.get("all_area").toString()
def temp = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000).longValue()
if (temp > 100000) {
row << ""
} else {
row << temp
}
} else {
if (dbo.get("all_area")) {
if (Double.valueOf(dbo.get("all_area")) > 100000) {
row << ""
} else {
row << dbo.get("all_area")
}
} else {
row << ""
}
}
if (dbo.get("build_area") && Double.valueOf(dbo.get("build_area")) > 1000) {
row << dbo.get("build_area")
} else {
row << ""
}
row << "pa101" // 小区类型
row << "" // 是否新盘
if (dbo.get("virescence_percent") != null && dbo.get("cubage_percent") != null) {
// 绿化率
if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) < 1) {
row << Double.valueOf(dbo.get("virescence_percent")) * 100
} else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) == 1) {
row << Double.valueOf(dbo.get("virescence_percent")) * 10
} else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) >= 70) {
row << 65
} else {
row << Double.valueOf(dbo.get("virescence_percent"))
}
if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 7 && Double.valueOf(dbo.get("cubage_percent")) < 8)) {
row << 6.2
} else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 8 && Double.valueOf(dbo.get("cubage_percent")) < 9)) {
row << 6.3
} else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 9 && Double.valueOf(dbo.get("cubage_percent")) < 10)) {
row << 6.5
} else if (dbo.get("cubage_percent") && Double.valueOf(dbo.get("cubage_percent")) >= 10) {
row << 6.7
} else {
row << dbo.get("cubage_percent") // 容积率
}
} else {
def temp = CalcCubageAndVirescence.findResult(dbo.lat / 10000000000.0d, dbo.lng / 10000000000.0d, 500, sameRegionDistricts.get(dbo.region), dbo.name, dbo._id)
if (temp[2] && Double.valueOf(temp[2]) < 1) {
row << Double.valueOf(temp[2]) * 100
} else if (temp[2] && Double.valueOf(temp[2]) == 1) {
row << Double.valueOf(temp[2]) * 10
} else if (temp[2] && Double.valueOf(temp[2]) >= 70) {
row << 65
} else {
row << temp[2] // 绿化率
}
if (temp[1] && (Double.valueOf(temp[1]) >= 7 && Double.valueOf(temp[1]) < 8)) {
row << 6.2
} else if (temp[1] && (Double.valueOf(temp[1]) >= 8 && Double.valueOf(temp[1]) < 9)) {
row << 6.3
} else if (temp[1] && (Double.valueOf(temp[1]) >= 9 && Double.valueOf(temp[1]) < 10)) {
row << 6.5
} else if (temp[1] && Double.valueOf(temp[1]) >= 10) {
row << 6.7
} else {
row << temp[1] // 容积率
}
}
if (dbo.get("car_site_count") != null) {
if (Long.valueOf(dbo.get("car_site_count")) > 2000 || Long.valueOf(dbo.get("car_site_count")) < 100) {
row << ""
} else {
row << dbo.get("car_site_count") // 车位数量
}
} else {
row << "" // 车位数量
}
row << "" // 介绍
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
map.put(dbo.get("_id").toString(), row)
} catch(Exception e) {
} finally {
latch.countDown()
}
}
})
})
latch.await()
return map
};
/**
* 处理特殊字符
* @param dbo
*/
static def dealSpecialChar(DBObject dbo) {
Set<String> keySet = dbo.keySet()
for (String key : keySet) {
if (String.isInstance(dbo.get(key))) {
dbo.put(key, pattern.matcher(dbo.get(key)).replaceAll(""))
}
}
};
static def getSeaweedTypeCode(def value) {
def ret
if (StringUtils.isNotBlank(value)) {
ret = seaweedTypeMap.get(value)
if (StringUtils.isBlank(ret)) {
ret = "99" // 其他
}
} else {
ret = ""
}
return ret
}
}
@@ -0,0 +1,28 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
class Temp {
static main (args){
DBObject query = new BasicDBObject()
query.append("cat", "district").append("visible", 0).append("city", "±±¾©")
DBObject filter = new BasicDBObject()
filter.append("build_structure", 1)
Set<String> set = new HashSet<>()
DbUtils.seaweed.find(query, filter).batchSize(10000).each({ dbo ->
if (dbo.build_structure) {
set.add(dbo.build_structure)
}
})
set.each {val ->
println(val)
}
}
}
@@ -0,0 +1,55 @@
package com.fangjia.script.temp.shenjian.util
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
/**
* ÇøÓò¹¤¾ßÀà
*/
class DistrictUtils {
static main(args) {
}
static def getAllDistricts(def cat, def city, def sameRegionDistricts){
DBObject query = new BasicDBObject();
query.append("cat", cat).append("visible", 0).append("city", city)
DBObject keys = new BasicDBObject().append("city", 1).append("region", 1).append("cubage_percent", 1)
.append("virescence_percent", 1).append("lat2", 1).append("lng2", 1)
.append("finish_building_date", 1).append("name", 1).append("_id", 1);
DSCursor cursor = DbUtils.seaweed.find(query, keys).batchSize(10000)
Iterator<DBObject> iters = cursor.iterator()
def result = []
while(iters.hasNext()) {
DBObject obj = iters.next()
def map = [:];
map.city = obj.city
map.cubage_percent = obj.cubage_percent;
map.virescence_percent = obj.virescence_percent;
map.lat2 = obj.lat2
map.lng2 = obj.lng2
map.region = obj.region
map.finish_building_date = obj.finish_building_date
map.name = obj.name
map._id = obj._id
result << map;
if (sameRegionDistricts.get(obj.region)) {
def temp = sameRegionDistricts.get(obj.region)
temp << map
sameRegionDistricts.put(obj.region, temp)
} else {
def temp = []
temp << map
sameRegionDistricts.put(obj.region, temp)
}
}
return result;
}
}