From 113f2e8c506154773725fc21e51f431b0d19ad5e Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Mon, 17 May 2021 20:58:57 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=8F=E5=8C=BAmaster=E5=88=86=E6=94=AF?= =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/company/AddCompany.groovy | 71 +++++ .../shenjian/company/DevelopCompany.groovy | 33 ++ .../shenjian/company/PropertyCompany.groovy | 34 +++ .../temp/shenjian/dto/SearchAddressDTO.groovy | 16 + .../fill/CalcCubageAndVirescence.groovy | 116 +++++++ .../shenjian/seaweed/BuildStructure.groovy | 112 +++++++ .../temp/shenjian/seaweed/BuildType.groovy | 129 ++++++++ .../temp/shenjian/seaweed/BuildYear.groovy | 91 ++++++ .../temp/shenjian/seaweed/CollectTest.groovy | 87 ++++++ .../temp/shenjian/seaweed/Company.groovy | 155 ++++++++++ .../script/temp/shenjian/seaweed/Fee.groovy | 112 +++++++ .../temp/shenjian/seaweed/HousePurpose.groovy | 109 +++++++ .../temp/shenjian/seaweed/HouseType.groovy | 99 ++++++ .../temp/shenjian/seaweed/Position.groovy | 78 +++++ .../temp/shenjian/seaweed/Seaweed.groovy | 288 ++++++++++++++++++ .../script/temp/shenjian/seaweed/Temp.groovy | 28 ++ .../temp/shenjian/util/DistrictUtils.groovy | 55 ++++ 17 files changed, 1613 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy new file mode 100644 index 000000000..1c8d2f70f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy @@ -0,0 +1,71 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils; + +/** + * + * 新增公司 + * + * @author shenjian + * @since 2021-03-16 + */ +class AddCompany { + + static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + + static final Set developCompanySet = new HashSet() + static final Set propertyCompanySet = new HashSet() + + static final Set actualDevelopCompanySet = new HashSet() + static final Set actualPropertyCompanySet = new HashSet() + + static { + develop_company.find().batchSize(10000).each { obj -> + developCompanySet.add(obj.company) + } + property_company.find().batchSize(10000).each { obj -> + propertyCompanySet.add(obj.company) + } + } + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0).append("city", "哈尔滨") + DbUtils.seaweed.find(query, + new BasicDBObject("developer", 1).append("estate_company", 1)) + .batchSize(10000).each {obj -> + actualDevelopCompanySet.add(obj.developer) + actualPropertyCompanySet.add(obj.estate_company) + } + + def developCompanySaves = [] + def propertyCompanySaves = [] + for (String name : actualDevelopCompanySet) { + if (StringUtils.isBlank(name) || developCompanySet.contains(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + developCompanySaves << item + } + for (String name : actualPropertyCompanySet) { + if (StringUtils.isBlank(name) || propertyCompanySet.contains(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + propertyCompanySaves << item + } + + + develop_company.insert(developCompanySaves) + property_company.insert(propertyCompanySaves) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy new file mode 100644 index 000000000..fb1e94c2b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy @@ -0,0 +1,33 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils; + +/** + * + * 开发商公司 + * + * @author shenjian + * @since 2021-03-13 + */ +class DevelopCompany { + + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136") + static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + + static void main(String[] args) { + def companySaves = [] + List result = seaweed.distinct("developer") + for (String name : result) { + if (StringUtils.isBlank(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + companySaves << item + } + develop_company.insert(companySaves) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy new file mode 100644 index 000000000..dca212b14 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy @@ -0,0 +1,34 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils + +/** + * 物业公司 + * + * @author shenjian + * @since 2021-03-13 + */ +class PropertyCompany { + + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136") + static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + + static void main(String[] args) { + // 物业公司 + def companySaves = [] + List result = seaweed.distinct("estate_company") + for (String name : result) { + if (StringUtils.isBlank(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + companySaves << item + } + property_company.insert(companySaves) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy new file mode 100644 index 000000000..01b1c1f27 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy @@ -0,0 +1,16 @@ +package com.fangjia.script.temp.shenjian.dto + +/** + * 匹配返回结构体 + */ +class SearchAddressDTO { + + String code; + String msg; + Result result; + + class Result { + String street + String streetNumber + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy new file mode 100644 index 000000000..ada6e667b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.fill + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.temp.shenjian.util.DistrictUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.GeoHelper +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 根据周围500米容积率与绿化率平均值推算小区 + * + * @author shenjian* + * @since 2021-03-13 + */ +@Slf4j +class CalcCubageAndVirescence { + + static main(args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("CalcCubageAndVirescence") + log.info("开始根据周围500米容积率与绿化率平均值推算小区") + def cities = ['北京', '重庆', '哈尔滨'] + for (def city : cities) { + def sameRegionDistricts = [:] + def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts); + def result = [] + districts.each { obj -> + if (obj.finish_building_date) { + int year = CommonUtils.getBuildYear(obj.finish_building_date) + if (year > 2000) { + if (!(obj.cubage_percent && obj.virescence_percent)) { + def temp = this.findResult(obj.lat2 / 10000000000.0d, obj.lng2 / 10000000000.0d, 500, sameRegionDistricts.get(obj.region), obj.name, obj._id) + result << temp + } + + } + } + } + def headers = ["小区ID", "容积率", "绿化率"] + FileUtils.batchCreateFile(result, "d:\\数据\\小区\\补充容积绿化" + city + ".csv", headers, "UTF-8") + } + stopWatch.stop() + log.info("完成根据周围500米容积率与绿化率平均值推算, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + } + + static def findResult(lat, lng, radius, districts, region, id) { + def avgMap = [:] + def ret = [id] + if (lat && lng) { + def cubagePercentArr = [] + def virescencePercentArr = [] + def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, radius) + for (def row : districts) { + if (row.lat2 && row.lng2) { + def lat2 = row.lat2 / 10000000000.0; + def lng2 = row.lng2 / 10000000000.0; + if (lat2 >= map.minLat && lat2 <= map.maxLat && lng2 >= map.minLng && lng2 <= map.maxLng) { + def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2) + if (distance <= radius) { + if (row.cubage_percent) { + cubagePercentArr << row.cubage_percent; + } + if (row.virescence_percent) { + virescencePercentArr << row.virescence_percent + } + } + } + } + } + try { + // 计算平均容积率 + if (cubagePercentArr) { + double sum = 0 + double count = 0.0d + for (def m : cubagePercentArr) { + sum = sum + Double.valueOf(m) + count++ + } + if (sum) { + double avgCubagePercent = sum / count + ret << DataHandleUtils.getDoubleRound(2, avgCubagePercent) + } + } else { + ret << "" + } + + // 计算平均绿化率 + if (virescencePercentArr) { + double sum = 0 + double count = 0.0d + for (def m : virescencePercentArr) { + sum += Double.valueOf(m) + count++ + } + if (sum) { + def avgVirescencePercent = sum / (count) + ret << DataHandleUtils.getDoubleRound(2, avgVirescencePercent) + } + } else { + ret << "" + } + log.info("找到可以优化的小区" + region + "周围平均值为:容积率" + ret[1] + ", 绿化率" + ret[2]) + } catch(Exception ex) { + log.error("容积率或绿化率不标准,无法解析") + } + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy new file mode 100644 index 000000000..278446f50 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑结构 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildStructure { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map buildStructureMap = new HashMap() + static { + buildStructureMap.put("钢结构", "3") + buildStructureMap.put("钢、钢筋混凝土结构", "1") + buildStructureMap.put("钢筋混凝土结构", "1") + buildStructureMap.put("钢混", "1") + buildStructureMap.put("钢混结构", "1") + buildStructureMap.put("混合结构", "11") + buildStructureMap.put("钢筋混凝土", "11") + buildStructureMap.put("砖木结构", "10") + buildStructureMap.put("其他结构", "11") + buildStructureMap.put("砖混结构", "2") + buildStructureMap.put("框架结构", "4") + buildStructureMap.put("框架剪力墙结构", "5") + buildStructureMap.put("剪力墙结构", "6") + buildStructureMap.put("筒体结构", "7") + buildStructureMap.put("框架筒体结构", "8") + buildStructureMap.put("筒中筒", "9") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区建筑结构表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("build_structure", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑结构","备注", "创建时间", "更新时间", "来源公司"] + def seaweedCount = [21596, 17053, 5410]; + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑结构\\bigdata_commbuildstru_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑结构表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << this.getBuildStructure(dbo.get("build_structure")) // 建筑结构 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("build_structure"))) { + rows << row + count.incrementAndGet() + } + }) + return rows + }; + + static def getBuildStructure(def houseStructure) { + def ret + if (StringUtils.isNotBlank(houseStructure)) { + ret = buildStructureMap.get(houseStructure) + if (StringUtils.isBlank(ret)) { + ret = "11" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy new file mode 100644 index 000000000..408b87f48 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy @@ -0,0 +1,129 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑类型表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildType { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map buildTypeMap = new HashMap() + static { + buildTypeMap.put("多层", "0104") + buildTypeMap.put("小高层", "0103") + buildTypeMap.put("高层", "0199") + buildTypeMap.put("超高层", "0199") + buildTypeMap.put("花园洋房", "0199") + buildTypeMap.put("老洋房", "0199") + buildTypeMap.put("四合院", "0199") + buildTypeMap.put("独栋", "0105") + buildTypeMap.put("双拼", "0199") + buildTypeMap.put("联排", "0106") + buildTypeMap.put("叠加", "0107") + buildTypeMap.put("老式别墅洋房", "0199") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区建筑类型表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑类型","备注", "创建时间", "更新时间", "来源公司"]; + def seaweedCount = [21596, 17053, 5410]; + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑类型\\bigdata_commbuildtype_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + if (StringUtils.isNotBlank(dbo.get("build_type"))) { + Set set = this.getBuildTypeCode(dbo.get("build_type")) + for (String type : set) { + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << type // 建筑类型 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + rows << row + } + if (set.size() > 0) { + count.incrementAndGet() + } + + } + }) + return rows + } + + + static Set getBuildTypeCode(def buildType) { + Set set = new HashSet<>() + if (StringUtils.isNotBlank(buildType)) { + String[] types = buildType.split(",") + for (String type : types) { + set.add(buildTypeMap.get(type)) + } + if (set.isEmpty()) { + set.add("0199") // 其他 + } + } + return set + } + + static List getBuildTypeCodeSeq(def buildType) { + List list = new ArrayList() + if (StringUtils.isNotBlank(buildType)) { + String[] types = buildType.split(",") + for (String type : types) { + list.add(buildTypeMap.get(type)) + } + if (list.isEmpty()) { + list.add("0199") // 其他 + } + } + return list + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy new file mode 100644 index 000000000..cee15c27c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.text.SimpleDateFormat +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑年代表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildYear { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildYear") + log.info("开始导出小区建筑年代表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("finish_building_date", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑年代","备注", "创建时间", "更新时间", "来源公司"] + + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑年代\\bigdata_commbuildyear_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑年代表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + Integer year = CommonUtils.getBuildYear(dbo.get("finish_building_date")) // 建筑年代 + if (year > 1900) { + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << year + row << "" // 备注 + + Integer createYear = CommonUtils.getBuildYear(DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy")) + if (year > createYear) { + row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date()); + row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date()) + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } + row << "fjw" // 来源公司 + + rows << row + count.incrementAndGet() + } + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy new file mode 100644 index 000000000..e5cbce7af --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy @@ -0,0 +1,87 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.ExcelUtil +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * @author shenjian + * @since 2021-03-13 + */ +class CollectTest { + + static void main(String[] args) { + def cites = ["北京", "重庆", "哈尔滨"]; + for (def city : cites) { + /** + * _id 小区ID + * name 小区名 + * alias 小区曾用名 + */ + def districts = this.findAllCityDistricts(city, "district", + ["city", "_id", "region", "name", "alias"]) + print(districts) + } + } + + static def findAllCityDistricts(def city, def cat, def fields) { + DBObject q = new BasicDBObject(); + q.put("visible", 0) + q.put("cat", cat) + q.put("region", "密云") + if (city) { + q.put("city", city) + } + + DBObject keys = new BasicDBObject(); + if (fields) { + for (def field : fields) { + keys.put(field, 1) + } + } + + DSCursor cursor = CommonDatautils.getDataStoreConnection().find(q, keys).batchSize(4000) + def districtKeys = [] + def rows = []; + rows << ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积", + "小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"] + + for (def dbo : cursor) { + def row = []; + row << null // TODO + row << dbo.get("_id") + row << dbo.get("name") + row << dbo.get("alias") + row << null // TODO + row << dbo.get("street_community") // 街道 TODO 拆分为街道门牌号 + row << dbo.get("street_community") // 门牌号 + rows << dbo.get("address") + rows << dbo.get("all_area") + rows << dbo.get("build_area") + + rows << dbo.get("land_property") + ExcelUtils.write("d:\\数据\\小区表.xlsx", rows, "小区") + + StringBuilder data = new StringBuilder() + Set keySet = dbo.keySet() + keySet.forEach({key -> + if ("alias" == key) { + def value = dbo.get(key) + if (value != null) { + value = value[0] + } + data.append(value) + } else { + data.append(dbo.get(key)) + } + data.append("|@|") + }) + districtKeys << StringUtils.substringBeforeLast(data.toString(), "|@|") + }; + return districtKeys; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy new file mode 100644 index 000000000..87b609daa --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang3.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区公司表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Company { + + private static ExecutorService executorService = Executors.newFixedThreadPool(2); + + // 物业公司 + static final DBCollection PROPERTY_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + // 开发商 + static final DBCollection DEVELOP_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + + static final propertyCompany = new HashMap() + static final developCompany = new HashMap() + + static { + PROPERTY_COMPANY_COLLECTION.find().forEach({company -> + propertyCompany.put(company.get("company"), company.get("_id").toString()) + }) + + DEVELOP_COMPANY_COLLECTION.find().forEach({company -> + developCompany.put(company.get("company"), company.get("_id").toString()) + }) + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Company") + log.info("开始导出小区公司表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_company", 1).append("developer", 1) + .append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID","公司ID","公司类型", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger developCount = new AtomicInteger(0) + AtomicInteger estateCount = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, developCount, estateCount) + println("城市" + city + " " + cityId + "物业数量为" + estateCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((estateCount.get() * 100) / seaweedCount[i])) ) + println("城市" + city + " " + cityId + "开发商数量为" + developCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((developCount.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\公司\\bigdata_commcomp_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区公司表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger developCount, AtomicInteger estateCount) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + def rowCanSave = true + def row1CanSave = true + row << cityId // 城市ID + row << dbo.get("_id").toString() + def propertyCompany = dbo.get("estate_company") + if (StringUtils.isNotBlank(propertyCompany)) { + row << this.getPropertyCompanyId(propertyCompany) // 公司ID + row << "SV" // 公司类型(物业管理公司SV、 房地产开发商DV ) + } else { + rowCanSave = false + } + + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + def row1 = [] + row1 << cityId + row1 << dbo.get("_id").toString() + def developerCompany = dbo.get("developer") + if (StringUtils.isNotBlank(developerCompany)) { + row1 << this.getDevelopCompanyId(developerCompany) // 公司ID + row1 << "DV" // 公司类型(物业管理公司SV、 房地产开发商DV ) + } else { + row1CanSave = false + } + + row1 << "" // 备注 + row1 << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row1 << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row1 << "fjw" // 来源公司 + if (rowCanSave) { + rows << row + estateCount.incrementAndGet() + } + if (row1CanSave) { + rows << row1 + developCount.incrementAndGet() + } + }) + return rows + } + + /** + * 获取开发商公司ID + * @param company 公司名称 + * @return + */ + static def getDevelopCompanyId(String company) { + if (StringUtils.isBlank(company)) { + return "" + } + return developCompany.get(company) + } + + /** + * 获取物业公司ID + * @param company 公司名称 + * @return + */ + static def getPropertyCompanyId(String company) { + if (StringUtils.isBlank(company)) { + return "" + } + return propertyCompany.get(company) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy new file mode 100644 index 000000000..614da8edb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区费用表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Fee { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Fee") + log.info("开始导出小区费用表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_company", 1).append("estate_charge", 1) + .append("estate_type2", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID", "公司ID", "费用", "房屋用途","建筑类型", "小区费用类型", "费用说明", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\费用\\bigdata_commfee_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区费用表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def estate_charge = dbo.get("estate_charge") + if (estate_charge && estate_charge.toString().indexOf("~") != -1) { + String[] arr = estate_charge.toString().split("~") + estate_charge = ((Double.valueOf(arr[0]) + Double.valueOf(arr[1])) / 2.0D).toString() + } + if (estate_charge == '-' || estate_charge == '无') { + estate_charge = "" + } + if (StringUtils.isNotBlank(dbo.get("estate_company")) && StringUtils.isNotBlank(estate_charge) + && StringUtils.isNotBlank(dbo.get("estate_type2")) && StringUtils.isNotBlank(dbo.get("build_type"))) { + List buildTypes = BuildType.getBuildTypeCodeSeq(dbo.get("build_type")) + Set savedBuildTypeSet = new HashSet<>() + boolean isFirst = true + for (String buildType : buildTypes) { + if (savedBuildTypeSet.contains(buildType)) { + continue + } + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() // 小区ID + row << Company.getPropertyCompanyId(dbo.get("estate_company")) // 公司ID + + // 处理 db.getCollection('seaweed').find({'estate_charge': '3.3~4.3'}, {'estate_charge': 1}) + // 特殊费用 3.3~4.3 + if (isFirst) { + row << estate_charge // 费用 + isFirst = false + count.incrementAndGet() + } else { + row << "" + break + } + row << HousePurpose.getHousePurposeCode(dbo.get("estate_type2")) // 房屋用途 + row << buildType // 建筑类型 + row << "1" // 小区费用类型 物业费1 电梯费2 垃圾费3 + row << dbo.get("") // 费用说明 + row << dbo.get("") // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + rows << row + + savedBuildTypeSet.add(buildType) + } + } + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy new file mode 100644 index 000000000..ba8dc7c2e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy @@ -0,0 +1,109 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.StringUtil +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区房屋用途表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class HousePurpose { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map housePurposeMap = new HashMap() + static { + housePurposeMap.put("普通住宅", "11") + housePurposeMap.put("别墅", "11") + housePurposeMap.put("商住", "13") + housePurposeMap.put("其他", "99") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区房屋用途表表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_type2", 1).append("household_count", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","房屋用途","户数", "备注", "创建时间", "更新时间", "来源公司"]; + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\房屋用途\\bigdata_commhouseuse_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区房屋用途表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << getHousePurposeCode(dbo.get("estate_type2")) + + if (dbo.get("household_count") != null) { + if (Long.valueOf(dbo.get("household_count")) < 50) { + row << "" // 户数 + } else { + row << dbo.get("household_count") + } + } else { + row << "" // 户数 + } + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + if (dbo.get("m_date")) { + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") + } + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("estate_type2"))) { + rows << row + } + }) + return rows + }; + + static def getHousePurposeCode(def housePurpose) { + def ret + if (StringUtils.isNotBlank(housePurpose)) { + ret = housePurposeMap.get(housePurpose) + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy new file mode 100644 index 000000000..8e77760a9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy @@ -0,0 +1,99 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区房屋类型表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class HouseType { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map houseTypeMap = new HashMap() + static { + houseTypeMap.put("普通住宅", "01") + houseTypeMap.put("商住", "07") + houseTypeMap.put("别墅", "08") + houseTypeMap.put("其他", "99") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("HouseType") + log.info("开始导出小区房屋类型表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_type2", 1).append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID","小区房屋类型","备注", "创建时间", "更新时间", "来源公司"] + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\房屋类型\\bigdata_commhousetype_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区房屋类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << this.getHouseTypeCode(dbo.get("estate_type2")) // 小区房屋类型 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + if (dbo.get("m_date")) { + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") + } + + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("estate_type2"))) { + rows << row + } + }) + return rows + }; + + static def getHouseTypeCode(def houseType) { + def ret + if (StringUtils.isNotBlank(houseType)) { + ret = houseTypeMap.get(houseType) // 房屋类型 + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy new file mode 100644 index 000000000..269f01ab3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区地理位置表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Position { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Location") + log.info("开始导出小区地理位置表结构表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("lng", 1).append("lat", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","物体类型","物体ID","位置类型", "序号", "经度", "纬度", "海拔", "备注", "创建时间", "更新时间", "来源公司"] + + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\地理位置\\bigdata_position_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区地理位置表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << "5" // 物体类型 小区5 + row << dbo.get("_id").toString() // 物体ID + row << "1" // 位置类型 中心点1 + row << "1" // 序号 + row << DataHandleUtils.getDoubleRound(6, dbo.get("lng") / 10000000000d) // 经度 + row << DataHandleUtils.getDoubleRound(6, dbo.get("lat") / 10000000000d) // 纬度 + row << "" // 海拔 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + rows << row + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy new file mode 100644 index 000000000..0c2f16c13 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy @@ -0,0 +1,288 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.temp.shenjian.dto.SearchAddressDTO +import com.fangjia.script.temp.shenjian.fill.CalcCubageAndVirescence +import com.fangjia.script.temp.shenjian.util.DistrictUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.regex.Pattern + +/** + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Seaweed { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + private static HttpRequest http = new HttpRequest() + private static final String TOKEN = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + private static Map seaweedTypeMap = new HashMap(); + private static Pattern pattern = Pattern.compile("\\s*|\t|\r|\n"); // 取消空格换行等 + private static Map cityDistrictMap = new HashMap(); + + static { + seaweedTypeMap.put("普通住宅", "pa101") + seaweedTypeMap.put("别墅", "pa101") + seaweedTypeMap.put("商住", "pa101") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Seaweed") + log.info("开始导出小区公司表") + //def cites = ["北京", "重庆", "哈尔滨"] + //def seaweedCount = [21596, 17053, 5410]; + def cites = ["哈尔滨"] + def seaweedCount = [5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("name", 1).append("alias", 1).append("street_community", 1) + .append("address", 1).append("all_area", 1).append("build_area", 1).append("region", 1) + .append("estate_type2", 1).append("virescence_percent", 1).append("cubage_percent", 1) + .append("car_site_count", 1).append("c_date", 1).append("m_date", 1) + .append("lng", 1).append("lat", 1) + def headers = ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积", + "小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + final t = i; + def city = cites[t] + def sameRegionDistricts = [:] + def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts) + BasicDBObject cityDistrictInfo = FangJiaAreaMapUtils.convertToOfficial(city) + def cityId = cityDistrictInfo.get("city_code") // 城市ID + final def map = this.findAllCityDistricts(city, "district", cityId, filter, sameRegionDistricts) + executorService.execute(new Runnable() { + @Override + void run() { + def result = [] + AtomicInteger virescenceCount = new AtomicInteger(0) // 绿化率 + AtomicInteger cubageCount = new AtomicInteger(0) // 容积率 + AtomicInteger allAreaCount = new AtomicInteger(0) + AtomicInteger buildAreaCount = new AtomicInteger(0) + for (def key : map.keys()) { + result << map.get(key) + if (map.get(key) && map.get(key)[8] != "") { + allAreaCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[9] != "") { + buildAreaCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[12] != "") { + virescenceCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[13] != "") { + cubageCount.incrementAndGet() + } + } + println("城市" + city + " " + cityId + "绿化率数量为" + virescenceCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((virescenceCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "容积率数量为" + cubageCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((cubageCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "占地面积数量为" + allAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((allAreaCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "建筑面积数量为" + buildAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((buildAreaCount.get() * 100) / seaweedCount[t])) ) + FileUtils.batchCreateFile(result, "d:\\数据\\小区\\bigdata_community_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + fixedThreadPool.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, def sameRegionDistricts) { + def map = new ConcurrentHashMap(); + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + def total = DbUtils.seaweed.find(query, filter).count() + final CountDownLatch latch = new CountDownLatch(total) + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + fixedThreadPool.execute(new Runnable() { + @Override + void run() { + try { + dealSpecialChar(dbo) + def row = [] + + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << dbo.get("name") + if (dbo.get("alias") != null && dbo.get("alias")[0] != null) { + row << pattern.matcher(dbo.get("alias")[0]).replaceAll("") + } else { + row << "" + } + def districtId + if (StringUtils.isBlank(cityDistrictMap.get(dbo.get("region")))) { + districtId = FangJiaAreaMapUtils.convertToOfficial(city, dbo.get("region")).get("district_code") + // 行政区ID + cityDistrictMap.put(dbo.get("region"), districtId) + } else { + districtId = cityDistrictMap.get(dbo.get("region")) + } + row << districtId + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city); + params.put("address", dbo.get("address")); + // 暂时不用解析 +// String s = http.get("http://open.fangjia.com/address/standardize", params, "utf-8"); +// SearchAddressDTO searchAddress = com.alibaba.fastjson.JSONObject.parseObject(s, SearchAddressDTO.class) +// if (searchAddress.code == '200') { +// //row << searchAddress.result.street // 街道 +// row << "" +// row << searchAddress.result.streetNumber // 门牌号 +// } else { +// row << "" +// row << "" +// } + row << "" // 街道 + row << "" // 门牌号 + + row << dbo.get("address") + + if (dbo.get("all_area") && dbo.get("all_area").toString().indexOf("公顷") != -1) { + def val = dbo.get("all_area").toString() + def temp = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000).longValue() + if (temp > 100000) { + row << "" + } else { + row << temp + } + + } else { + if (dbo.get("all_area")) { + if (Double.valueOf(dbo.get("all_area")) > 100000) { + row << "" + } else { + row << dbo.get("all_area") + } + } else { + row << "" + } + } + if (dbo.get("build_area") && Double.valueOf(dbo.get("build_area")) > 1000) { + row << dbo.get("build_area") + } else { + row << "" + } + + row << "pa101" // 小区类型 + row << "" // 是否新盘 + + if (dbo.get("virescence_percent") != null && dbo.get("cubage_percent") != null) { + // 绿化率 + if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) < 1) { + row << Double.valueOf(dbo.get("virescence_percent")) * 100 + } else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) == 1) { + row << Double.valueOf(dbo.get("virescence_percent")) * 10 + } else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) >= 70) { + row << 65 + } else { + row << Double.valueOf(dbo.get("virescence_percent")) + } + + if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 7 && Double.valueOf(dbo.get("cubage_percent")) < 8)) { + row << 6.2 + } else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 8 && Double.valueOf(dbo.get("cubage_percent")) < 9)) { + row << 6.3 + } else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 9 && Double.valueOf(dbo.get("cubage_percent")) < 10)) { + row << 6.5 + } else if (dbo.get("cubage_percent") && Double.valueOf(dbo.get("cubage_percent")) >= 10) { + row << 6.7 + } else { + row << dbo.get("cubage_percent") // 容积率 + } + } else { + def temp = CalcCubageAndVirescence.findResult(dbo.lat / 10000000000.0d, dbo.lng / 10000000000.0d, 500, sameRegionDistricts.get(dbo.region), dbo.name, dbo._id) + if (temp[2] && Double.valueOf(temp[2]) < 1) { + row << Double.valueOf(temp[2]) * 100 + } else if (temp[2] && Double.valueOf(temp[2]) == 1) { + row << Double.valueOf(temp[2]) * 10 + } else if (temp[2] && Double.valueOf(temp[2]) >= 70) { + row << 65 + } else { + row << temp[2] // 绿化率 + } + + if (temp[1] && (Double.valueOf(temp[1]) >= 7 && Double.valueOf(temp[1]) < 8)) { + row << 6.2 + } else if (temp[1] && (Double.valueOf(temp[1]) >= 8 && Double.valueOf(temp[1]) < 9)) { + row << 6.3 + } else if (temp[1] && (Double.valueOf(temp[1]) >= 9 && Double.valueOf(temp[1]) < 10)) { + row << 6.5 + } else if (temp[1] && Double.valueOf(temp[1]) >= 10) { + row << 6.7 + } else { + row << temp[1] // 容积率 + } + } + if (dbo.get("car_site_count") != null) { + if (Long.valueOf(dbo.get("car_site_count")) > 2000 || Long.valueOf(dbo.get("car_site_count")) < 100) { + row << "" + } else { + row << dbo.get("car_site_count") // 车位数量 + } + } else { + row << "" // 车位数量 + } + + row << "" // 介绍 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + map.put(dbo.get("_id").toString(), row) + } catch(Exception e) { + + } finally { + latch.countDown() + } + } + }) + }) + latch.await() + return map + }; + + /** + * 处理特殊字符 + * @param dbo + */ + static def dealSpecialChar(DBObject dbo) { + Set keySet = dbo.keySet() + for (String key : keySet) { + if (String.isInstance(dbo.get(key))) { + dbo.put(key, pattern.matcher(dbo.get(key)).replaceAll("")) + } + } + }; + + static def getSeaweedTypeCode(def value) { + def ret + if (StringUtils.isNotBlank(value)) { + ret = seaweedTypeMap.get(value) + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy new file mode 100644 index 000000000..d85ddf81f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy @@ -0,0 +1,28 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +class Temp { + + static main (args){ + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0).append("city", "北京") + + DBObject filter = new BasicDBObject() + filter.append("build_structure", 1) + + Set set = new HashSet<>() + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({ dbo -> + if (dbo.build_structure) { + set.add(dbo.build_structure) + } + }) + + set.each {val -> + println(val) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy new file mode 100644 index 000000000..03d95a32c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * 区域工具类 + */ +class DistrictUtils { + + static main(args) { + } + + static def getAllDistricts(def cat, def city, def sameRegionDistricts){ + DBObject query = new BasicDBObject(); + query.append("cat", cat).append("visible", 0).append("city", city) + + DBObject keys = new BasicDBObject().append("city", 1).append("region", 1).append("cubage_percent", 1) + .append("virescence_percent", 1).append("lat2", 1).append("lng2", 1) + .append("finish_building_date", 1).append("name", 1).append("_id", 1); + + DSCursor cursor = DbUtils.seaweed.find(query, keys).batchSize(10000) + Iterator iters = cursor.iterator() + def result = [] + while(iters.hasNext()) { + DBObject obj = iters.next() + + def map = [:]; + map.city = obj.city + map.cubage_percent = obj.cubage_percent; + map.virescence_percent = obj.virescence_percent; + map.lat2 = obj.lat2 + map.lng2 = obj.lng2 + map.region = obj.region + map.finish_building_date = obj.finish_building_date + map.name = obj.name + map._id = obj._id + result << map; + if (sameRegionDistricts.get(obj.region)) { + def temp = sameRegionDistricts.get(obj.region) + temp << map + sameRegionDistricts.put(obj.region, temp) + } else { + def temp = [] + temp << map + sameRegionDistricts.put(obj.region, temp) + } + } + return result; + } +}