Merge branch 'script_gj' into 'master'

Script gj

See merge request !217
This commit is contained in:
2021-12-10 10:47:55 +08:00
115 changed files with 11235 additions and 50 deletions
+31 -26
View File
@@ -1,5 +1,5 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.fangjia</groupId>
<artifactId>fangjia-script</artifactId>
@@ -62,7 +62,7 @@
</resources>
<pluginManagement>
<plugins>
<!--This plugin's configuration is used to store Eclipse m2e settings
<!--This plugin's configuration is used to store Eclipse m2e settings
only. It has no influence on the Maven build itself. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
@@ -141,11 +141,11 @@
<artifactId>activemq-pool</artifactId>
<version>5.5.0</version>
</dependency>
<dependency>
<groupId>cn.emay</groupId>
<artifactId>emayclient</artifactId>
<version>4.3</version>
</dependency>
<dependency>
<groupId>cn.emay</groupId>
<artifactId>emayclient</artifactId>
<version>4.3</version>
</dependency>
<dependency>
<groupId>org.apache.activemq</groupId>
<artifactId>activemq-core</artifactId>
@@ -366,18 +366,20 @@
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>3.10-FINAL</version>
<version>3.17</version>
<!-- <version>3.10-FINAL</version>-->
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.10-FINAL</version>
<exclusions>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
</exclusions>
<version>3.17</version>
<!-- <version>3.10-FINAL</version>-->
<exclusions>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- <dependency>
@@ -476,18 +478,21 @@
<artifactId>aliyun-java-sdk-address-purification</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.eclipsesource.j2v8</groupId>
<artifactId>j2v8_win32_x86_64</artifactId>
<version>4.6.0</version>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>2.2.8</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.20</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.msgpack</groupId>
<artifactId>msgpack</artifactId>
<version>0.6.12</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>elastic.co</id>
<url>https://artifacts.elastic.co/maven</url>
</repository>
</repositories>
</project>
@@ -390,7 +390,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
}
println name +" "+houses.size()+" last:"+rets.size()
println commId +" "+houses.size()+" last:"+rets.size()
return rets;
}
@@ -0,0 +1,183 @@
package com.fangjia.script.temp.shenjian
import com.alibaba.excel.EasyExcel
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import java.util.concurrent.*
/**
* 成交详情导出
*
* @author shenjian* @since 2021/3/29
*/
class DealPriceDetailExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017)
static final Map retMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
static main(args) {
def cities = ["北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州",
"合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台",
"徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁",
"南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华",
"昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城",
"芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州",
"连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口",
"西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"]
// def cities = ["常州"]
def headArr = ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量", "小区均价", "比率"]
List<String> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
}
CountDownLatch latch = new CountDownLatch(cities.size())
DBObject filter = new BasicDBObject()
filter.put("city", 1)
filter.put("region", 1)
filter.put("district_name", 1)
filter.put("total_price", 1)
filter.put("trade_date", 1)
filter.put("avg_price", 1)
filter.put("area", 1)
filter.put("direction", 1)
filter.put("fitment", 1)
filter.put("height", 1)
filter.put("floor", 1)
filter.put("unit_num", 1)
filter.put("house_num", 1)
filter.put("room_num", 1)
filter.put("hall", 1)
filter.put("room", 1)
filter.put("toilet", 1)
for (def city : cities) {
final currCity = city
executorService.execute(new Runnable() {
@Override
void run() {
try {
final def ret = new CopyOnWriteArrayList()
// 小区
def map = [:]
DBUtils.seaweed.find( new BasicDBObject("city", currCity).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
.batchSize(10000).each {obj ->
if (obj.region != null) {
map.put(obj.region + obj.name, obj)
if (obj.alias) {
// 小区可能存在多个别名
for (def a : obj.alias) {
map.put(obj.region + a, obj)
}
}
}
}
//ret << ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量"]
DBObject query = new BasicDBObject();
query.put("city", currCity)
query.put("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", currCity).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
.batchSize(10000).count())
fangjiaConn.find(query, filter).batchSize(10000).each { tempObj ->
final obj = tempObj
districtExecutorService.execute(new Runnable() {
@Override
void run() {
try {
if (map.get(obj.region + obj.district_name)) {
def row = []
row[0] = currCity
row[1] = obj.region
row[2] = obj.district_name
row[3] = obj.total_price
row[4] = obj.avg_price
if (obj.trade_date) {
row[5] = DateUtils.formatDateByPattern(obj.trade_date, "yyyy-MM-dd")
}
row[6] = obj.area
row[7] = obj.direction
row[8] = obj.fitment
row[9] = obj.height
row[10] = obj.floor
row[11] = obj.unit_num
row[12] = obj.house_num
row[13] = obj.room_num
row[14] = obj.hall
row[15] = obj.room
row[16] = obj.toilet
def objOur = map.get(obj.region + obj.district_name)
calcOurAvgPrice(currCity, obj.trade_date, obj.avg_price, objOur, row)
ret.add(row)
} else {
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
}
} catch (Exception ex) {
println("出现异常111111.。。。。。。。。。。。")
ex.printStackTrace()
} finally {
districtLatch.countDown()
}
}
})
}
districtLatch.await()
EasyExcel.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx").head(head).sheet("成交信息").doWrite(ret);
//ExcelUtils.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx", ret, "成交信息")
println("当前处理城市" + currCity + " " + ret.size())
} catch (Exception ex) {
println("出现异常22222.。。。。。。。。。。。")
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
executorService.shutdown()
districtExecutorService.shutdown()
}
private static calcOurAvgPrice(def city, def tradeDate, def avgPrice, def objOur, def row) {
// 最相近一周的该小区均价
Date nextWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, 7)
Date firstWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, -7)
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
dbObject.put("region", objOur.region)
dbObject.put("district_name", objOur.name)
// 不可用append,会被覆盖
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
Double weekendAvgPrice = 0
dbCursor.each { o ->
weekendAvgPrice = o.avg_price
}
if (weekendAvgPrice == 0) {
println("周均价未找到:" + city + " " + objOur.region + " " + objOur.name)
row[17] = ""
row[18] = ""
} else {
row[17] = weekendAvgPrice
row[18] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(weekendAvgPrice) / Double.valueOf(avgPrice))
}
}
}
@@ -0,0 +1,158 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCursor
import org.apache.commons.lang3.time.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 模型级联项目涉及城市
*
* @author shenjian
* @since 2021/3/23
*/
class PPE15Export {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static Map retMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
static main(args) {
def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州",
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
//def cities = ["扬州"]
def result = []
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
CountDownLatch latch = new CountDownLatch(cities.size())
for (def cityTemp : cities) {
final city = cityTemp
println("当前处理城市:" + city)
executorService.execute(new Runnable() {
@Override
void run() {
try {
AtomicInteger totalCount = new AtomicInteger(0)
AtomicInteger count = new AtomicInteger(0)
def row = []
row[0] = city
// 小区
def map = [:]
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
.batchSize(10000).each {obj ->
if (obj.region != null) {
map.put(obj.region + obj.name, obj)
if (obj.alias) {
// 小区可能存在多个别名
for (def a : obj.alias) {
map.put(obj.region + a, obj)
}
}
}
}
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
.batchSize(10000).count())
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
.batchSize(10000).each { obj ->
districtExecutorService.execute(new Runnable() {
@Override
void run() {
try {
if (map.get(obj.region + obj.district_name)) {
def objOur = map.get(obj.region + obj.district_name)
Double avgPrice = obj.avg_price
Date tradeDate = obj.trade_date
// 最相近一周的该小区均价
Date nextWeekDate = DateUtils.addDays(tradeDate, 7)
Date firstWeekDate = DateUtils.addDays(tradeDate, -7)
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
dbObject.put("region", obj.region)
dbObject.put("district_name", objOur.name)
// 不可用append,会被覆盖
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
dbObject.put("region", obj.region)
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
// DBCursor dbCursor = collection.find(new BasicDBObject("city", city).append("region", objOur.region).append("district_name", objOur.name)
// .append("weekend", new BasicDBObject('$lte', nextWeekDate))
// .append("weekend", new BasicDBObject('$gte', firstWeekDate))).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
Double weekendAvgPrice = 0
dbCursor.each { o ->
weekendAvgPrice = o.avg_price
}
if (weekendAvgPrice == 0) {
println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name)
} else {
boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15
if (ppe15) {
count.incrementAndGet()
}
}
totalCount.incrementAndGet()
} else {
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
}
} catch (Exception ex) {
ex.printStackTrace()
} finally {
districtLatch.countDown()
}
}
})
};
districtLatch.await()
row[6] = totalCount.get()
row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)"
retMap.put(city, row)
println("当前处理城市" + city + " " + row)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
};
latch.await()
for (def city: cities) {
if (retMap.get(city)) {
result << retMap.get(city)
} else {
def row = []
row[0] = city
row[6] = 0
row[7] = 0
result << row
}
}
ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
executorService.shutdown()
districtExecutorService.shutdown()
}
}
@@ -0,0 +1,130 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCursor
import org.apache.commons.lang3.time.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 模型级联项目涉及城市
*
* @author shenjian
* @since 2021/3/23
*/
class PPE15ExportTest {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static Map retMap = new ConcurrentHashMap()
static main(args) {
// def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州",
// "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
// "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
// "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
// "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
// "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
// "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
// "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def cities = ["常州"]
def result = []
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
for (def cityTemp : cities) {
final city = cityTemp
println("当前处理城市:" + city)
try {
AtomicInteger totalCount = new AtomicInteger(0)
AtomicInteger count = new AtomicInteger(0)
def row = []
row[0] = city
// 小区
final def map = [:]
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
.batchSize(10000).each {obj ->
if (obj.region != null) {
map.put(obj.region + obj.name, obj)
if (obj.alias) {
for (def a : obj.alias) {
map.put(obj.region + a, obj)
}
}
}
}
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
int ppe15Count = 0
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24"))))
.batchSize(10000).count())
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24"))))
.batchSize(10000).each { obj ->
try {
if (map.get(obj.region + obj.district_name)) {
def objOur = map.get(obj.region + obj.district_name)
Double avgPrice = obj.avg_price
Date tradeDate = obj.trade_date
// 最相近一周的该小区均价
Date nextWeekDate = DateUtils.addDays(tradeDate, 7)
Date firstWeekDate = DateUtils.addDays(tradeDate, -7)
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
dbObject.put("region", obj.region)
dbObject.put("district_name", objOur.name)
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
dbObject.put("region", obj.region)
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
Double weekendAvgPrice = 0
dbCursor.each { o ->
weekendAvgPrice = o.avg_price
}
if (weekendAvgPrice == 0) {
println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name)
} else {
boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15
if (ppe15) {
count.incrementAndGet()
}
}
totalCount.incrementAndGet()
} else {
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
}
} catch (Exception ex) {
ex.printStackTrace()
} finally {
districtLatch.countDown()
}
}
districtLatch.await()
row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)"
retMap.put(city, row)
println("当前处理城市" + city + " " + row)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
}
};
for (def city: cities) {
if (retMap.get(city)) {
result << retMap.get(city)
} else {
result << 0
}
}
ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
}
}
@@ -0,0 +1,153 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 上海房屋租金导出
*
* @author shenjian
* @since 2021/03/22
*/
class RentExport {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static Map retMap = new ConcurrentHashMap()
static main(args) {
def result = []
result << ["城市","区域","小区名称","房型","租金(月)","计算方式", "经度", "纬度"]
int count = DBUtils.seaweed.find(new BasicDBObject("city", "上海"),
new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1))
.batchSize(10000).count()
AtomicInteger size = new AtomicInteger(count)
println("上海小区" + size.get())
CountDownLatch latch = new CountDownLatch(count)
DBUtils.seaweed.find(new BasicDBObject("city", "上海"),
new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1))
.batchSize(10000).each {obj ->
executorService.submit(new Runnable() {
@Override
void run() {
try {
DBObject query = new BasicDBObject()
query.append("city", "上海").append("district_name", obj.name).append("region", obj.region)
.append("end_month_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2020-09-22")))
.append("house_type", new BasicDBObject('$lt', 5))
DBObject filter = new BasicDBObject()
filter.append("region", 1).append("city", 1).append("district_name", 1).append("avg_price", 1)
.append("house_type", 1).append("sample_enum", 1).append("_id", 0).append("end_month_date", 1)
buildResult(query, filter, result, obj.lng, obj.lat)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
println("*********************" + size.decrementAndGet())
}
}
})
}
latch.await()
result.addAll(retMap.values())
com.fangjia.script.util.ExcelUtils.write("G:\\数据\\建行\\租金\\租金.xlsx", result, "租金");
executorService.shutdown()
}
static boolean buildResult (def query, def filter, def result, def lng, def lat) {
collection.find(query, filter).batchSize(10000).sort(new BasicDBObject("end_month_date", -1).append("house_type", 1)).each {obj ->
def row = []
row[0] = obj.city
row[1] = obj.region
row[2] = obj.district_name
row[3] = obj.house_type
row[4] = obj.avg_price
row[5] = obj.sample_enum
row[6] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lng / 10000000000d)
row[7] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lat / 10000000000d)
if (!retMap.containsKey(obj.district_name + obj.region + obj.house_type)) {
// 判断价格是否合理
if (isPriceReasonable(obj.district_name, obj.region, obj.house_type, obj.avg_price)) {
retMap.put(obj.district_name + obj.region + obj.house_type, row)
println(retMap.size())
} else {
println("当前价格不合理,被过滤掉" + obj.district_name + obj.region + obj.house_type)
}
}
}
}
static boolean isPriceReasonable(def district_name, def region, def house_type, def avg_price) {
if (retMap.contains(district_name + region + "1")) {
printf("一居室" + retMap.get(district_name + region + "1")[4])
}
if (retMap.contains(district_name + region + "2")) {
printf("二居室" + retMap.get(district_name + region + "2")[4])
}
if (retMap.contains(district_name + region + "3")) {
printf("三居室" + retMap.get(district_name + region + "3")[4])
}
println("当前居室" + house_type + ", 价格为" + avg_price)
switch (house_type) {
case 2:
println(retMap.get(district_name + region + "1"))
if (retMap.get(district_name + region + "1")) {
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
break;
case 3:
if (retMap.get(district_name + region + "1")) {
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
if (retMap.get(district_name + region + "2")) {
double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4])
if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
break;
case 4:
if (retMap.get(district_name + region + "1")) {
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
if (retMap.get(district_name + region + "2")) {
double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4])
if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
if (retMap.get(district_name + region + "3")) {
double threeAvgPrice = Double.valueOf(retMap.get(district_name + region + "3")[4])
if (threeAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
break;
}
return true
}
}
@@ -8,6 +8,7 @@ import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang3.time.DateFormatUtils
/**
* 抓取总量监控
@@ -37,7 +38,16 @@ class SeawaterMonitor extends BaseBatch {
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
static void main(String[] args) {
runBatch(new SeawaterMonitor(), args)
//runBatch(new SeawaterMonitor(), args)
//monitorDayCrawlReasonable()
def results = []
results << ["日期", "网站", "出售", "出租"]
for (int i = 1; i < 90; i++) {
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i).getTime()
def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime()
countDayCrawlBySite(startDate, endDate, results)
}
ExcelUtils.write("f://数据//爬虫//分析.xlsx", results, "sheet1")
}
/**
@@ -125,6 +135,52 @@ class SeawaterMonitor extends BaseBatch {
return ""
}
/**
* 统计近90天更新量超过40万的天数
* @param args
*/
static def monitorDayCrawlReasonable() {
def map = [:]
for (int i = 1; i < 90; i++) {
def date = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i)
def startDate = date.getTime();
def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime()
def total = countDayCrawl(startDate, endDate)
if (total >= expectNumber) {
map.put(DateUtil.formatToYYYYMMDD(date), total)
}
}
println("近90天更新数量超过" + expectNumber + "的天数为" + map.size() + ",分别为:" )
for (def key : map.keySet()) {
println(key + " : " + map.get(key))
}
}
/**
* 统计每个网站时间段范围内的更新量
*/
static def countDayCrawlBySite(def startDate, def endDate, def results) {
def sites = ["lianjia.com", "ke.com", "fang.com", "58.com", "anjuke.com"]
def sql = "select count(*) as total, root, cat from seawater where m_date >= '${startDate}' and m_date < '${endDate}' group by root, cat"
def map = [:]
ElasticUtils.executeSqlQuery(sql,{data->
map.put(data.root + "|" + data.cat, data.total)
});
for (def site : sites) {
results << [DateFormatUtils.format(startDate, "yyyy-MM-dd"), site, map.get(site + "|sell"), map.get(site + "|rent")]
}
return results
};
static def countDayCrawl(def startDate, def endDate) {
def total = 0
def sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}'"
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
return total
};
@Override
void process(Object args) {
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
@@ -0,0 +1,15 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
/**
* @author shenjian* @since 2021/5/18
*/
class Temp {
static void main(String[] args) {
// 121.431870,31.208610
List list = GaoDeUtils.searchPoi("上海", "浦东新区新场实验小学新城校区")
print(list)
}
}
@@ -0,0 +1,92 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
/**
* 模型级联项目涉及城市
*
* @author shenjian
* @since 2021/3/23
*/
class TotalTestExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static Map retMap = new ConcurrentHashMap()
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
static main(args) {
def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州",
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
//def cities = ["贵阳"]
def result = []
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
//CountDownLatch latch = new CountDownLatch(cities.size())
for (def city : cities) {
try {
def row = []
row[0] = city
// 小区
def map = [:]
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
.batchSize(10000).each {obj ->
if (obj.region != null) {
map.put(obj.region + obj.name, 1)
if (obj.alias) {
for (def a : obj.alias) {
map.put(obj.region + a, 1)
}
}
}
}
// 测试数据量(小区近一年成交量)
int tradeCount = 0
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
.batchSize(10000).each { obj ->
if (map.get(obj.region + obj.district_name)) {
tradeCount++
} else {
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
}
}
row[6] = tradeCount
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
//int ppe15 =
//row[7] =
result << row
println("当前处理城市" + city + " " + row)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
//latch.countDown()
}
}
//latch.await()
ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
}
}
@@ -0,0 +1,91 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
/**
* 模型级联项目涉及城市
*
* @author shenjian
* @since 2021/3/23
*/
class ValueRateCityInfoExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static Map retMap = new ConcurrentHashMap()
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
static main(args) {
def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州",
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
//def cities = ["重庆","郑州", "长沙", "北京"]
def result = []
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
//CountDownLatch latch = new CountDownLatch(cities.size())
for (def city : cities) {
try {
def row = []
row[0] = city
// 小区数量
int seaweedCount = DBUtils.seaweed.count( new BasicDBObject("city", city).append("cat", "district").append("visible", 0))
// 可估值小区数量
int valuateCount = collection.find(new BasicDBObject("city", city)
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
).count()
// 有样本估值数量及占比
int haveSampleCount = collection.find(new BasicDBObject("city", city)
.append("value_type", 2)
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
).count()
// 无样本估值数量及占比
int notHaveSampleCount = valuateCount - haveSampleCount
// 可信度大于0.5的数量
int creditCount = collection.find(new BasicDBObject("city", city)
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
.append("credit", new BasicDBObject('$gte', 0.5d))
).count()
row[1] = seaweedCount
row[2] = valuateCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(valuateCount * 100) / Double.valueOf(seaweedCount)) + "%)"
if (valuateCount != 0) {
row[3] = haveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(haveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)"
row[4] = notHaveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(notHaveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)"
row[5] = creditCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(creditCount * 100) / Double.valueOf(valuateCount)) + "%)"
}
result << row
println("当前处理城市" + city + " " + row)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
//latch.countDown()
}
}
//latch.await()
ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.0-test.xlsx", result, "Sheet1")
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.temp.shenjian.address
import com.alibaba.excel.EasyExcel
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 导出标准化输出结果
*
* @author shenjian
* @since 2021/6/22
*/
class ExportStandardResult {
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017)
static void main(String[] args) {
List<List<String>> data = new ArrayList<>()
def map = new ConcurrentHashMap()
// 查询条件
DBObject query = new BasicDBObject('status', 0).append('flag', '浙商疑问地址复测')
CountDownLatch latch = new CountDownLatch(conn136.count(query).intValue())
conn136.find(query).batchSize(10000).each {
objT ->
final obj = objT
executorService.submit(new Runnable() {
@Override
void run() {
try {
def ret = []
ret << obj.format_province
ret << obj.format_city
ret << obj.format_region
ret << obj.format_dev
ret << obj.format_township
ret << obj.format_community
ret << obj.format_group
ret << obj.format_street
ret << obj.format_streetNumber
ret << obj.format_name
ret << obj.format_buildingNumber
ret << obj.format_unitNumber
ret << obj.format_roomNumber
ret << obj.format_floor
ret << obj.format_address
map.put(obj.order, ret)
} catch (Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
for (int i = 0; i < map.size(); i++) {
data.add(map.get(i))
}
def headArr = ["省","市","区","开发区","街道乡镇","居委会/社区", "组","路","路号","名称","楼栋", "单元","室号","楼层","优化后标准化地址"]
List<String> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
}
EasyExcel.write("F:\\工作\\基础建设\\标准化\\浙商疑问地址复测.xlsx").head(head).sheet("测试数据").doWrite(data);
executorService.shutdown()
}
}
@@ -0,0 +1,99 @@
package com.fangjia.script.temp.shenjian.address
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 地址标准化测试
*
* @author shenjian
* @since 2021/6/22
*/
class StandardTest {
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017)
static final String IP = "192.168.0.19:39080"
static void main(String[] args) {
AtomicLong count = new AtomicLong(0)
// 查询条件
DBObject query = new BasicDBObject('status', 1).append('flag', '浙商疑问地址复测')
CountDownLatch latch = new CountDownLatch(conn136.count(query).intValue())
conn136.find(query).batchSize(3000).each {objT ->
final def obj = objT
executorService.submit(new Runnable() {
@Override
void run() {
try {
def address = obj.address
def city = obj.city == null ? "" : obj.city
HttpRequest request = new HttpRequest()
String result = request.get("http://" + IP + "/FangJiaAddressService/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8")
JSONObject object = JSONObject.parse(result);
if (object.get("status") == 0) {
JSONObject jsonResult = (JSONObject) object.get("result");
def format_province = jsonResult.get("province")
def format_city = jsonResult.get("city")
def format_region = jsonResult.get("district")
def format_dev = jsonResult.get("dev")
def format_township = jsonResult.get("township")
def format_community = jsonResult.get("community")
def format_group = jsonResult.get("group")
def format_street = jsonResult.get("street")
def format_subStreet = jsonResult.get("subStreet")
def format_streetNumber = jsonResult.get("streetNumber")
def format_name = jsonResult.get("name")
def format_buildingNumber = jsonResult.get("buildingNumber")
def format_unitNumber = jsonResult.get("unitNumber")
def format_floor = jsonResult.get("floor")
def format_roomNumber = jsonResult.get("roomNumber")
def format_address = jsonResult.get("formatAddress")
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0)
.append("format_province", format_province)
.append("format_city", format_city)
.append("format_region", format_region)
.append("format_dev", format_dev)
.append("format_township", format_township)
.append("format_community", format_community)
.append("format_group", format_group)
.append("format_street", format_street)
.append("format_subStreet", format_subStreet)
.append("format_streetNumber", format_streetNumber)
.append("format_name", format_name)
.append("format_buildingNumber", format_buildingNumber)
.append("format_unitNumber", format_unitNumber)
.append("format_floor", format_floor)
.append("format_roomNumber", format_roomNumber)
.append("format_address", format_address)))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "标准化输出异常")))
}
} catch(Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
count.incrementAndGet()
if (count.intValue() % 1000 == 0) {
Thread.sleep(5000)
println(count)
}
}
}
})
};
latch.await();
executorService.shutdown()
}
}
@@ -0,0 +1,53 @@
package com.fangjia.script.temp.shenjian.business
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
/**
* 获取商圈地标
*
* @author shenjian
* @since 2021/4/13
*/
class BusinessDianpingAddr {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
static void main(String[] args) {
def cities = ["苏州", "西安", "合肥"]
for (def city : cities) {
DBObject query = new BasicDBObject()
query.put("city", city)
collection.find(query).batchSize(300).each {obj ->
def id = obj._id
if (obj.lng == null || obj.lat == null) {
// 查询坐标位置,并更新
def location = this.parserByGaode(city, obj.region + obj.name + obj.address)
if (location != null) {
DBObject updateField = new BasicDBObject()
updateField.put("lng", location.lng)
updateField.put("lat", location.lat)
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', updateField), false, true)
}
}
}
}
}
static def parserByGaode(def city, def address) {
JSONObject json = GaoDeApiUtils.geo(city, address)
if (json && json.status == "1") {
def ret = json.geocodes
if (ret) {
def str = ret.get(0).location
def level = ret.get(0).level
def arrs = str.split(",")
return ["lng": arrs[0], "lat": arrs[1]]
}
}
}
}
@@ -0,0 +1,98 @@
package com.fangjia.script.temp.shenjian.business
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
/**
* 商圈地标导出
*
* @author shenjian
* @since 2021/4/13
*/
class BusinessDianpingExport {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static void main(String[] args) {
def cityCodes = ["320500","610100","340100"];
def cities = ["苏州", "西安", "合肥"]
def headers = ["城市", "区域", "商圈名称", "地标名称(3-5个)", "地标地址(与地标名称对应)", "地标经度(与地标名称对应)", "地标纬度(与地标名称对应)", "备注", "创建时间", "更新时间", "来源公司"];
for (int i = 0; i < cities.size(); i++) {
final def city = cities[i];
def result = []
// 获取所有商圈信息
def businessNames = collection.distinct("name", new BasicDBObject("city", city))
// 获取商圈下3个地标
for (def name : businessNames) {
List<String> landmarks = new ArrayList<>()
List<String> addresses = new ArrayList<>()
List<String> lngs = new ArrayList<>()
List<String> lats = new ArrayList<>()
String region = ""
String c_date
String m_date
int landmarkLen = 0;
collection.find(new BasicDBObject("city", city).append("name", name)).sort(new BasicDBObject("order", 1)).each { obj ->
// 按人气取经纬度不为空的地标
if (landmarks.size() < 5) {
// 特殊标题去重
if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1
|| String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1
|| String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1
|| String.valueOf(obj.landmark).indexOf(",") != -1 || String.valueOf(obj.landmark).indexOf("商圈附近") != -1 || String.valueOf(obj.landmark).indexOf("】") != -1) {
} else {
// 防止导入导致字段过长,一个字符2个字节
def landmark = String.valueOf(obj.landmark).replaceAll(" ", "")
if (landmark.length() <= 30 && landmarkLen + String.valueOf(landmark).length() <= 100) {
landmarkLen += String.valueOf(landmark).length()
landmarks.add(landmark)
addresses.add(obj.address)
lngs.add(obj.lng)
lats.add(obj.lat)
if (StringUtils.isBlank(region)) {
region = obj.region
c_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss");
m_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.m_date, "yyyy-MM-dd HH:mm:ss");
}
}
}
}
}
def ret = []
def city_code
def district_code
if (StringUtils.isNotBlank(region)) {
cityCollection.find(new BasicDBObject("fangjia_city", city).append("district", new BasicDBObject('$regex', region + "+")),
new BasicDBObject("city_code", 1).append("district_code", 1)).each {obj ->
city_code = obj.city_code
district_code = obj.district_code
}
}
ret[0] = city_code
ret[1] = district_code
ret[2] = name
ret[3] = StringUtils.join(landmarks, ",")
ret[4] = StringUtils.join(addresses, ",")
ret[5] = StringUtils.join(lngs, ",")
ret[6] = StringUtils.join(lats, ",")
ret[7] = ""
ret[8] = c_date
ret[9] = m_date
ret[10] = "fjw"
if (district_code != null) {
result << ret
} else {
println("未找到商圈地标:" + name)
}
}
FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8")
}
}
}
@@ -0,0 +1,51 @@
package com.fangjia.script.temp.shenjian.business
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
/**
* 商圈数据修正
*
* @author shenjian
* @since 2021/4/13
*/
class BusinessDianpingModify {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
static void main(String[] args) {
def cityCodes = ["320500","610100","340100"];
def city = "合肥"
def origin = "政务区"
def now = "蜀山区"
collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj ->
collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true)
}
city = "苏州"
origin = "工业园区"
now = "吴中区"
collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj ->
collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true)
}
city = "苏州"
origin = "甪直"
now = "吴中区"
collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj ->
collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true)
}
city = "西安"
origin = "高新区"
now = "雁塔区"
collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj ->
collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true)
}
}
}
@@ -0,0 +1,71 @@
package com.fangjia.script.temp.shenjian.company
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils;
/**
*
* ÐÂÔö¹«Ë¾
*
* @author shenjian
* @since 2021-03-16
*/
class AddCompany {
static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136")
static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136")
static final Set developCompanySet = new HashSet()
static final Set propertyCompanySet = new HashSet()
static final Set actualDevelopCompanySet = new HashSet()
static final Set actualPropertyCompanySet = new HashSet()
static {
develop_company.find().batchSize(10000).each { obj ->
developCompanySet.add(obj.company)
}
property_company.find().batchSize(10000).each { obj ->
propertyCompanySet.add(obj.company)
}
}
static void main(String[] args) {
DBObject query = new BasicDBObject()
query.append("cat", "district").append("visible", 0).append("city", "¹þ¶û±õ")
DbUtils.seaweed.find(query,
new BasicDBObject("developer", 1).append("estate_company", 1))
.batchSize(10000).each {obj ->
actualDevelopCompanySet.add(obj.developer)
actualPropertyCompanySet.add(obj.estate_company)
}
def developCompanySaves = []
def propertyCompanySaves = []
for (String name : actualDevelopCompanySet) {
if (StringUtils.isBlank(name) || developCompanySet.contains(name)) {
continue;
}
def item = new BasicDBObject()
item.company = name
developCompanySaves << item
}
for (String name : actualPropertyCompanySet) {
if (StringUtils.isBlank(name) || propertyCompanySet.contains(name)) {
continue;
}
def item = new BasicDBObject()
item.company = name
propertyCompanySaves << item
}
develop_company.insert(developCompanySaves)
property_company.insert(propertyCompanySaves)
}
}
@@ -0,0 +1,33 @@
package com.fangjia.script.temp.shenjian.company
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang3.StringUtils;
/**
*
* ¿ª·¢É̹«Ë¾
*
* @author shenjian
* @since 2021-03-13
*/
class DevelopCompany {
static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136")
static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136")
static void main(String[] args) {
def companySaves = []
List result = seaweed.distinct("developer")
for (String name : result) {
if (StringUtils.isBlank(name)) {
continue;
}
def item = new BasicDBObject()
item.company = name
companySaves << item
}
develop_company.insert(companySaves)
}
}
@@ -0,0 +1,34 @@
package com.fangjia.script.temp.shenjian.company
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang3.StringUtils
/**
* ÎïÒµ¹«Ë¾
*
* @author shenjian
* @since 2021-03-13
*/
class PropertyCompany {
static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136")
static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136")
static void main(String[] args) {
// ÎïÒµ¹«Ë¾
def companySaves = []
List result = seaweed.distinct("estate_company")
for (String name : result) {
if (StringUtils.isBlank(name)) {
continue;
}
def item = new BasicDBObject()
item.company = name
companySaves << item
}
property_company.insert(companySaves)
}
}
@@ -0,0 +1,74 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.coredata.util.CollectionUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 重庆所有小区周边导出
*
* @author shenjian
* @since 2021/5/15
*/
class AllExport {
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018)
//3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息
static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204"
static def radius = 3000
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
DBObject query = new BasicDBObject()
query.put("city", "重庆")
query.put("cat", "district")
query.put("visible", 0)
DBObject filter = new BasicDBObject()
filter.put("region", 1)
filter.put("name", 1)
filter.put("lng", 1)
filter.put("lat", 1)
AtomicInteger count = new AtomicInteger(0)
DBUtils.seaweed.find(query, filter).batchSize(10000).each {district ->
executorService.execute(new Runnable() {
@Override
void run() {
def city = "重庆"
def name = district.name
def region = district.region
boolean exist = conn.count(new BasicDBObject("matched_city": city, "matched_region": region,"matched_name": name)) > 0 ? true : false
if (!exist) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.lng / 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.lat / 10000000000d)
List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "")
if (CollectionUtils.isNotEmpty(arounds)) {
for (BasicDBObject obj : arounds) {
obj.put("matched_city", city)
obj.put("matched_region", district.region)
obj.put("matched_name", district.name)
}
conn.insert(arounds)
}
println("当前处理小区" + name + " " + count.incrementAndGet())
} else {
println("小区已存在")
}
}
})
}
executorService.shutdown()
}
}
@@ -0,0 +1,81 @@
package com.fangjia.script.temp.shenjian.cqrcb
import cn.emay.mina.util.ConcurrentHashSet
import com.fangjia.script.coredata.util.CollectionUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区周围信息导出
*
* @author shenjian
* @since 2021/5/15
*/
class AroundExport {
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018)
//3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息
static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204"
static def radius = 3000
//static ExecutorService executorService = Executors.newFixedThreadPool(30)
// 高德地图中不存在周边的数据
// 重庆 江北 阳光丽苑
//重庆 江北 金田花苑
//重庆 巴南 巴滨一号璀璨滨江
//重庆 丰都 茅林沟移民新村
//重庆 江津 福城北苑
//重庆 巫山 摩天岭壹号
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 0)
AtomicInteger count = new AtomicInteger(0)
for (List<Object> districtT : data) {
final List<Object> district = districtT
// executorService.execute(new Runnable() {
// @Override
// void run() {
def city = district.get(0)
def region = district.get(1)
def name = district.get(2)
boolean exist = conn.count(new BasicDBObject("matched_city": city, "matched_region": region,"matched_name": name)) > 0 ? true : false
if (!exist) {
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("region", region)
query.put("cat", "district")
query.put("visible", 0)
query.put("name", name)
DBUtils.seaweed.find(query, new BasicDBObject("lat", 1).append("lng", 1)).limit(1).each {objT ->
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, objT.lng / 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, objT.lat / 10000000000d)
List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "")
if (CollectionUtils.isNotEmpty(arounds)) {
for (BasicDBObject obj : arounds) {
obj.put("matched_city", city)
obj.put("matched_region", region)
obj.put("matched_name", name)
}
conn.insert(arounds)
}
println("当前处理小区" + name + " " + count.incrementAndGet())
}
} else {
println("小区已存在")
}
// }
// })
}
//executorService.shutdown()
}
}
@@ -0,0 +1,189 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.alibaba.excel.EasyExcel
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.district.Distance
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.dto.AroundDTO
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.apache.commons.collections.map.HashedMap
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 周边数据导出新方式
*
* @author shenjian
* @since 2021/5/16
*/
class AroundExportByNewWay {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static def resultMap = new ConcurrentHashMap()
static def unMatchedMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
AtomicInteger count = new AtomicInteger(0)
CountDownLatch latch = new CountDownLatch(data.size())
for (List<Object> districtT : data) {
final List<Object> district = districtT
executorService.execute(new Runnable() {
@Override
void run() {
try {
def id = district.get(0)
def district_code = district.get(2)
def name = district.get(3)
List result = []
if (district.size() > 21 && district.get(20) != null && district.get(21) != null) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d)
calcAroundInfo(lat, lng, result, id)
} else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
DBObject query = new BasicDBObject()
query.put("city", district.get(22))
query.put("region", district.get(23))
query.put("cat", "district")
query.put("visible", 0)
query.put("name", district.get(24))
Set<String> set = new HashSet<>()
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj ->
if (obj.lng && obj.lat) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d)
calcAroundInfo(lat, lng, result, id, set)
}
}
} else {
def ret = []
ret[0] = id
ret[1] = district.get(1)
ret[2] = district_code
ret[3] = name
ret[4] = district.get(4)
unMatchedMap.put(id, ret)
}
resultMap.put(district_code + name, result)
println("当前处理" + count.incrementAndGet() )
} catch (Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
};
latch.await();
def headArr = ['IDB', '名称','类型', '距离']
List<String> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
}
Map<Integer, List> map = new HashedMap()
Integer key = 1;
def ret = []
AtomicInteger totalCount = new AtomicInteger()
for (List<Object> district : data) {
def district_code = district.get(2)
def name = district.get(3)
if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) {
totalCount.incrementAndGet()
List currRet = resultMap.get(district_code + name)
Collections.sort(currRet)
ret.addAll(currRet)
if (ret.size() > 700000) {
map.put(key++, ret)
ret = []
}
}
}
map.put(key++, ret)
//com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套");
for (Integer k : map.keySet()) {
//FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8")
EasyExcel.write("F:\\数据\\重庆农商行\\周边配套(文件"+ k + ")V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k));
}
print(totalCount.get() + " " + (totalCount.get() / data.size()))
printUnMatched(unMatchedMap)
executorService.shutdown()
}
static calcAroundInfo(def lat, def lng, def result, def id, def set) {
// 周围半径为3千米
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000)
DBObject q = new BasicDBObject()
q.put('status', 0)
q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng));
poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj ->
double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat)
if (distance <= 3000) {
if (distance == 0) {
distance = 50
}
if (!set.contains(obj.name)) {
AroundDTO aroundDTO = new AroundDTO()
aroundDTO.id = id
aroundDTO.name = obj.name
aroundDTO.type = formatType(obj.typecode)
aroundDTO.distance = distance.toInteger()
result.add(aroundDTO)
set.add(obj.name)
}
}
}
}
static printUnMatched(def unMatchedMap) {
def ret = []
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度']
for (def r :unMatchedMap.values()) {
ret << r;
}
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套");
}
static String formatType(String typeStr) {
def types = []
if (typeStr.indexOf("|") > -1) {
types = typeStr.split("\\|")
} else {
types =[typeStr]
}
for (def type : types) {
if (type.startsWith("09")) {
return "医院";
} else if (type.startsWith('1505')) {
return "地铁";
} else if (type.startsWith('1507')) {
return "公交";
} else if (type.startsWith("16")) {
return "银行";
} else if (type.startsWith("14")) {
return "学校";
} else if (type.startsWith("06")) {
return "商场"
}
}
return "未知"
}
}
@@ -0,0 +1,211 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.alibaba.excel.EasyExcel
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.district.Distance
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.dto.AroundDTO
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.apache.commons.collections.map.HashedMap
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 周边数据一千米配套数量统计
*
* @author shenjian
* @since 2021/5/16
*/
class AroundFormatExport {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static def resultMap = new ConcurrentHashMap()
static def unMatchedMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
AtomicInteger count = new AtomicInteger(0)
CountDownLatch latch = new CountDownLatch(data.size())
for (List<Object> districtT : data) {
final List<Object> district = districtT
executorService.execute(new Runnable() {
@Override
void run() {
try {
def id = district.get(0)
def district_code = district.get(2)
def name = district.get(3)
List result = []
if (district.size() > 21 && district.get(20) != null && district.get(21) != null) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d)
calcAroundInfo(lat, lng, result, id)
} else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
DBObject query = new BasicDBObject()
query.put("city", district.get(22))
query.put("region", district.get(23))
query.put("cat", "district")
query.put("visible", 0)
query.put("name", district.get(24))
Set<String> set = new HashSet<>()
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj ->
if (obj.lng && obj.lat) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d)
calcAroundInfo(lat, lng, result, id, set)
}
}
} else {
def ret = []
ret[0] = id
ret[1] = district.get(1)
ret[2] = district_code
ret[3] = name
ret[4] = district.get(4)
unMatchedMap.put(id, ret)
}
resultMap.put(district_code + name, result)
println("当前处理" + count.incrementAndGet() )
} catch (Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
};
latch.await();
def headArr = ['IDB', '数量']
List<String> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
}
Map<Integer, List> map = new HashedMap()
Integer key = 1;
def ret = []
AtomicInteger totalCount = new AtomicInteger()
for (List<Object> district : data) {
def id = district.get(0)
def district_code = district.get(2)
def name = district.get(3)
if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) {
totalCount.incrementAndGet()
List currRet = resultMap.get(district_code + name)
int gj = 0
int dt = 0
int yh = 0
int yy = 0
int sc = 0
int xx = 0
for (AroundDTO dto : currRet) {
if(dto.getType() == '公交') {
gj++
} else if(dto.getType() == '地铁') {
dt++
} else if(dto.getType() == '银行') {
yh++
} else if (dto.getType() == '医院') {
yy++
} else if(dto.getType() == '商场') {
sc++
} else if(dto.getType() == '学校') {
xx++
}
}
ret << [id, "公交:" + gj + ";" + "地铁:" + dt + ";银行:" + yh + ";医院:" + yy + ";商场:" + sc + ";学校:" + xx]
if (ret.size() > 700000) {
map.put(key++, ret)
ret = []
}
} else {
ret << [id, ""]
}
}
map.put(key++, ret)
//com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套");
for (Integer k : map.keySet()) {
//FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8")
EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k));
}
print(totalCount.get() + " " + (totalCount.get() / data.size()))
printUnMatched(unMatchedMap)
executorService.shutdown()
}
static calcAroundInfo(def lat, def lng, def result, def id, def set) {
// 周围半径为3千米
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
DBObject q = new BasicDBObject()
q.put('status', 0)
q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng));
poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj ->
double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat)
if (distance <= 1000) {
if (distance == 0) {
distance = 50
}
if (!set.contains(obj.name)) {
AroundDTO aroundDTO = new AroundDTO()
aroundDTO.id = id
aroundDTO.name = obj.name
aroundDTO.type = formatType(obj.typecode)
aroundDTO.distance = distance.toInteger()
result.add(aroundDTO)
set.add(obj.name)
}
}
}
}
static printUnMatched(def unMatchedMap) {
def ret = []
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度']
for (def r :unMatchedMap.values()) {
ret << r;
}
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套");
}
static String formatType(String typeStr) {
def types = []
if (typeStr.indexOf("|") > -1) {
types = typeStr.split("\\|")
} else {
types =[typeStr]
}
for (def type : types) {
if (type.startsWith("09")) {
return "医院";
} else if (type.startsWith('1505')) {
return "地铁";
} else if (type.startsWith('1507')) {
return "公交";
} else if (type.startsWith("16")) {
return "银行";
} else if (type.startsWith("14")) {
return "学校";
} else if (type.startsWith("06")) {
return "商场"
}
}
return "未知"
}
}
@@ -0,0 +1,61 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.atomic.AtomicInteger
/**
* 通过匹配出的小区核对经纬度
*
* @author shenjian
* @since 2021/5/19
*/
class CheckLatLngByDistrict {
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
for (List<Object> districtT : data) {
final List<Object> district = districtT
if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
DBObject query = new BasicDBObject()
query.put("city", district.get(22))
query.put("region", district.get(23))
query.put("cat", "district")
query.put("visible", 0)
query.put("name", district.get(24))
if (district.get(20) != null && district.get(21) != null) {
def originLng = district.get(20)
def originLat = district.get(21)
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj ->
if (obj.lng && obj.lat) {
if (originLng != obj.lng || originLat != obj.lat) {
def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d);
def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d);
originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d);
originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d);
double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng)
println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" )
}
} else {
println(district.get(24) + " 不存在经纬度")
}
}
} else {
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj ->
if (obj.lng && obj.lat) {
} else {
println(district.get(24) + " 不存在经纬度222222222")
}
}
}
}
}
}
}
@@ -0,0 +1,10 @@
package com.fangjia.script.temp.shenjian.cqrcb
/**
* @author shenjian
* @since 2021/5/15
*/
class Constants {
}
@@ -0,0 +1,106 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import com.mongodb.QueryOperators
/**
* 处理未匹配的小区
*
* @author shenjian
* @since 2021/5/17
*/
class DealUnMatched {
static void main(String[] args) {
// getLatLngByName()
// getLatLngByNumber()
//findExceptionDistance()
// B114862
def lng = 107.736591
def lat = 29.872502
// 周围半径为3千米
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 500)
DBObject q = new BasicDBObject()
q.put('city', '重庆')
q.put('status', 0)
q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng));
DBUtils.seaweed.find(q).batchSize(100).each {obj ->
println(obj)
}
}
static void findExceptionDistance() {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0)
for (List<Object> district : data) {
if (district.size() > 12) {
def lng = java.lang.Double.valueOf(district.get(8))
def lat = java.lang.Double.valueOf(district.get(9))
def lng1 = java.lang.Double.valueOf(district.get(11))
def lat1 = java.lang.Double.valueOf(district.get(12))
double distance = GaoDeUtils.getDistance(lat, lng, lat1, lng1)
if (distance > 500) {
println("距离异常" + district)
district.add(distance)
district.add("距离异常")
}
}
}
data.add(0, new ArrayList<Object>())
ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "周边配套")
}
static void getLatLngByNumber() {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",1)
for (List<Object> districtT : data) {
final List<Object> district = districtT
def city = district.get(1)
def district_code = district.get(2)
def name = district.get(3)
def number = district.get(4)
//def region = SeaweedUtils.getRegionByCode(district_code)
if (district.size() <= 5) {
if (number != null && number.size() > 5) {
def map = GaoDeUtils.parserByGaode(city, number)
if (map != null) {
district.add(map.get("lng"))
district.add(map.get("lat"))
district.add("通过楼栋得出")
}
} else {
println(district_code + " " + name + " " + number)
}
}
}
ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配1")
}
static void getLatLngByName() {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", 0)
for (List<Object> districtT : data) {
final List<Object> district = districtT
def city = district.get(1)
def district_code = district.get(2)
def name = district.get(3)
def number = district.get(4)
//def region = SeaweedUtils.getRegionByCode(district_code)
def map = GaoDeUtils.parserByGaode(city, name)
if (map != null) {
district.add(map.get("lng"))
district.add(map.get("lat"))
district.add("通过小区得出")
}
}
ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配")
}
}
@@ -0,0 +1,53 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 导入中学小学信息
*
* @author shenjian
* @since 2021/5/17
*/
class ImportSchool {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\重庆学校_0902.xlsx", 1)
List result = new ArrayList();
for (List<Object> school : data) {
try {
def city = school.get(0)
def region = school.get(1)
def name = school.get(2)
def address = school.get(4)
def type = school.get(6)
def lng = school.get(9)
def lat = school.get(10)
DBObject dbObject = new BasicDBObject()
dbObject.put("province", "重庆")
dbObject.put("city", city)
dbObject.put("region", region)
dbObject.put("name", name)
dbObject.put("address", address)
dbObject.put("type", type)
dbObject.put("typecode", "141200")
dbObject.put("lng", lng)
dbObject.put("lat", lat)
dbObject.put("status", 0)
dbObject.put("flag", "excel导入")
result.add(dbObject)
} catch(Exception ex) {
println(school)
ex.printStackTrace()
}
}
poiConn.insert(result)
}
}
@@ -0,0 +1,36 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import sun.swing.BakedArrayList
/**
* 标记不需要导出的数据
*
* @author shenjian
* @since 2021/5/16
*/
class MarkUnNeedExport {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 1)
for (List<Object> districtT : data) {
final List<Object> district = districtT
if (district.size() > 1) {
def type = district.get(0)
def status = district.get(1)
if (status == 1.0) {
poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 0)), false, true)
} else {
poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 1)), false, true)
}
}
}
}
}
@@ -0,0 +1,115 @@
package com.fangjia.script.temp.shenjian.cqrcb
import cn.emay.mina.util.ConcurrentHashSet
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* POI迁移
*
* @author shenjian
* @since 2021/5/16
*/
class MoveArroundTemp {
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018)
static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
Map<String, DBObject> map = new ConcurrentHashMap()
AtomicInteger count = new AtomicInteger()
int size = conn.count(new BasicDBObject("status", 1))
println(size)
CountDownLatch latch = new CountDownLatch(size)
conn.find(new BasicDBObject("status", 1)).batchSize(10000).each { objT ->
final def obj = objT
executorService.submit(new Runnable() {
@Override
void run() {
try {
if (map.get(obj.id) == null && conn43.count(new BasicDBObject("_id", obj.id)) <= 0) {
DBObject dbObject = new BasicDBObject()
dbObject.put("_id", obj.id)
dbObject.put("name", obj.name)
dbObject.put("type", obj.type)
dbObject.put("typecode", obj.typecode)
dbObject.put("address", obj.address)
dbObject.put("lng", Double.valueOf(obj.location.split(",")[0]))
dbObject.put("lat", Double.valueOf(obj.location.split(",")[1]))
dbObject.put("tel", obj.tel)
dbObject.put("province", obj.pname)
dbObject.put("city", obj.cityname)
dbObject.put("region", obj.adname)
dbObject.put("status", 2)
dbObject.put("flag", "待清洗")
map.put(obj.id, dbObject)
}
} catch(Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
println(count.incrementAndGet())
}
}
})
}
latch.await()
List ret = new ArrayList()
for (String key : map.keySet()) {
ret << map.get(key)
}
conn43.insert(ret)
executorService.shutdown()
}
static origin() {
// List<DBObject> list = Collections.synchronizedList(new ArrayList<DBObject>())
// List<String> ids = conn.distinct("id")
// CountDownLatch latch = new CountDownLatch(ids.size())
// for (String idT : ids) {
// final String id = idT
// executorService.submit(new Runnable() {
// @Override
// void run() {
// try {
// DBObject dbObject = new BasicDBObject()
// conn.find(new BasicDBObject("id", id)).limit(1).each { obj ->
// dbObject.put("_id", id)
// dbObject.put("name", obj.name)
// dbObject.put("type", obj.type)
// dbObject.put("typecode", obj.typecode)
// dbObject.put("address", obj.address)
// dbObject.put("lng", Double.valueOf(obj.location.split(",")[0]))
// dbObject.put("lat", Double.valueOf(obj.location.split(",")[1]))
// dbObject.put("tel", obj.tel)
// dbObject.put("province", obj.pname)
// dbObject.put("city", obj.cityname)
// dbObject.put("region", obj.adname)
// }
// list.add(dbObject)
// } catch(Exception ex) {
// ex.printStackTrace()
// } finally {
// latch.countDown()
// println("当前记录数" + list.size())
// }
// }
// })
// };
// latch.await();
// conn43.insert(list);
// list.clear();
// println("同步清理完成");
// executorService.shutdown()
}
}
@@ -0,0 +1,67 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.regex.Pattern
/**
* POI清理规则
*
* @author shenjian
* @since 2021/5/16
*/
class POIRule {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static void main(String[] args) {
}
static void changeSchool() {
// 小学校转为学校
poiConn.find(new BasicDBObject("typecode", "141203").append("status", 0)).batchSize(1000).each {obj ->
if (obj.name.indexOf('小学校') != -1 ) {
def id = obj._id
def region = obj.region
def name = obj.name.replace("小学校", '小学')
if (poiConn.count(new BasicDBObject("region": region, "name": name)) > 0) {
poiConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("status", 1)), false, true)
} else {
poiConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("name", name)), false, true)
}
}
}
}
static BasicDBObject getLikeStr(String findStr){
Pattern pattern = Pattern.compile('^.*' + findStr + '.*$', Pattern.CASE_INSENSITIVE);
return new BasicDBObject('$regex', pattern);
}
// 更新规则
// db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /[ABCDEFHG]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /购物服务;商场;免税品店/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;综合医院;卫生院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;学校/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /[1234]期/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /住院部/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /\//, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /专科医院/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /诊所/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
}
@@ -0,0 +1,55 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.coredata.util.CollectionUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* @author shenjian
* @since 2021/5/17
*/
class SavePoi {
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018)
static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
//3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息
static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204"
static def radius = 3000
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0)
AtomicInteger count = new AtomicInteger()
for (List<Object> listT : data) {
final list = listT
executorService.execute(new Runnable() {
@Override
void run() {
if (list.size() >= 17) {
def city = list.get(1)
def lng = Double.valueOf(list.get(11))
def lat = Double.valueOf(list.get(12))
List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "")
if (CollectionUtils.isNotEmpty(arounds)) {
for (BasicDBObject obj : arounds) {
obj.put("status", 1)
obj.put("flag", "距离异常入库")
}
conn.insert(arounds)
println(count.incrementAndGet())
}
}
}
})
};
executorService.shutdown()
}
}
@@ -0,0 +1,127 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.core.util.DateUtil
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.datastore.FieldValidatorUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 重庆农商行
* 开发商物业公司等小区信息导出
*
* @author shenjian
* @since 2021/5/14
*/
class SeaweedExport {
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static def map = new ConcurrentHashMap()
static void main(String[] args) {
def resultMap = new ConcurrentHashMap()
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
CountDownLatch latch = new CountDownLatch(data.size())
for (List<Object> districtT : data) {
final List<Object> district = districtT
executorService.execute(new Runnable() {
@Override
void run() {
try {
def city = district.get(1)
def district_code = district.get(2)
def name = district.get(3)
def matchedCity = district.get(22)
def matchedRegion = district.get(23)
def matchedName = district.get(24)
def region;
if (map.get("district_code") != null) {
region = map.get("district_code")
} else {
cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj ->
region = obj.fangjia_district
map.put("district_code", region)
}
}
if (matchedRegion == null) {
matchedRegion = region
matchedName = name
}
def ret = []
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("region", matchedRegion)
query.put("cat", "district")
query.put("visible", 0)
query.put("name", matchedName)
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("developer", 1)
filter.put("estate_company", 1)
filter.put("cubage_percent", 1) // 容积率
filter.put("virescence_percent", 1) // 绿化率
filter.put("alias", 1)
filter.put("car_site_count", 1)
filter.put("all_area", 1)
filter.put("loudong_count", 1)
filter.put("household_count", 1)
filter.put("estate_charge", 1)
filter.put("open_date", 1) // 开盘时间
filter.put("live_date", 1) // 交房时间
filter.put("lng", 1)
filter.put("lat", 1)
DbUtils.seaweed.find(query).limit(1).each { obj ->
ret[0] = obj.estate_company
ret[1] = obj.developer
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
ret[4] = SeaweedUtils.formatAlias(obj.alias)
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
ret[7] = ""
ret[8] = SeaweedUtils.formatAllArea(obj.all_area)
ret[9] = obj.loudong_count
ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count)
ret[11] = ""
ret[12] = obj.estate_charge
ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date)
ret[14] = obj.live_date
ret[15] = obj.lng
ret[16] = obj.lat
}
ret[17] = name
resultMap.put(district_code + name, ret)
} catch (Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
};
latch.await();
def ret = [];
ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"];
for (List<Object> district : data) {
def district_code = district.get(2)
def name = district.get(3)
ret << resultMap.get(district_code + name)
};
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商及物业信息.xlsx", ret, "开发商及物业信息");
executorService.shutdown()
}
}
@@ -0,0 +1,283 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.CityUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 58城市简写
*
* @author shenjian
* @since 2021/5/10
*/
class City58 {
static String cityList = "{\n" +
" \"安徽\":{\n" +
" \"合肥\":\"hf|837\",\"芜湖\":\"wuhu|2045\",\"蚌埠\":\"bengbu|3470\",\"阜阳\":\"fy|2325\",\"淮南\":\"hn|2319\",\n" +
" \"安庆\":\"anqing|3251\",\"宿州\":\"suzhou|3359\",\"六安\":\"la|2328\",\"淮北\":\"huaibei|9357\",\"滁州\":\"chuzhou|10266\",\n" +
" \"马鞍山\":\"mas|2039\",\"铜陵\":\"tongling|10285\",\"宣城\":\"xuancheng|5633\",\"亳州\":\"bozhou|2329\",\"黄山\":\"huangshan|2323\",\n" +
" \"池州\":\"chizhou|10260\",\"巢湖\":\"ch|10229\",\"和县\":\"hexian|10892\",\"霍邱\":\"hq|11226\",\"桐城\":\"tongcheng|11296\",\n" +
" \"宁国\":\"ningguo|5645\",\"天长\":\"tianchang|10273\",\"东至\":\"dongzhi|10262\",\"无为\":\"wuweixian|10232\"\n" +
" }\n" +
" ,\n" +
" \"福建\":{\n" +
" \"福州\":\"fz|304\",\"厦门\":\"xm|606\",\"泉州\":\"qz|291\",\"莆田\":\"pt|2429\",\"漳州\":\"zhangzhou|710\",\n" +
" \"宁德\":\"nd|7951\",\"三明\":\"sm|2048\",\"南平\":\"np|10291\",\"龙岩\":\"ly|6752\",\"武夷山\":\"wuyishan|10761\",\n" +
" \"石狮\":\"shishi|296\",\"晋江\":\"jinjiangshi|297\",\"南安\":\"nananshi|293\",\"龙海\":\"longhai|713\",\"上杭\":\"shanghangxian|6757\",\n" +
" \"福安\":\"fuanshi|7969\",\"福鼎\":\"fudingshi|7970\",\"安溪\":\"anxixian|7100\",\"永春\":\"yongchunxian|7101\",\"永安\":\"yongan|2133\",\n" +
" \"漳浦\":\"zhangpu|717\"\n" +
" }\n" +
" ,\n" +
" \"广东\":{\n" +
" \"深圳\":\"sz|4\",\"广州\":\"gz|3\",\"东莞\":\"dg|413\",\"佛山\":\"fs|222\",\"中山\":\"zs|771\",\n" +
" \"珠海\":\"zh|910\",\"惠州\":\"huizhou|722\",\"江门\":\"jm|629\",\"汕头\":\"st|783\",\"湛江\":\"zhanjiang|791\",\n" +
" \"肇庆\":\"zq|901\",\"茂名\":\"mm|679\",\"揭阳\":\"jy|927\",\"梅州\":\"mz|9389\",\"清远\":\"qingyuan|7303\",\n" +
" \"阳江\":\"yj|2284\",\"韶关\":\"sg|2192\",\"河源\":\"heyuan|10467\",\"云浮\":\"yf|10485\",\"汕尾\":\"sw|9449\",\n" +
" \"潮州\":\"chaozhou|10461\",\"台山\":\"taishan|11263\",\"阳春\":\"yangchun|8566\",\"顺德\":\"sd|8716\",\"惠东\":\"huidong|725\",\n" +
" \"博罗\":\"boluo|726\",\"海丰\":\"haifengxian|9444\",\"开平\":\"kaipingshi|634\",\"陆丰\":\"lufengshi|9456\"\n" +
" }\n" +
" ,\n" +
" \"广西\":{\n" +
" \"南宁\":\"nn|845\",\"柳州\":\"liuzhou|7133\",\"桂林\":\"gl|1039\",\"玉林\":\"yulin|2337\",\"梧州\":\"wuzhou|2046\",\n" +
" \"北海\":\"bh|10536\",\"贵港\":\"gg|6770\",\"钦州\":\"qinzhou|2335\",\"百色\":\"baise|10513\",\"河池\":\"hc|2340\",\n" +
" \"来宾\":\"lb|10552\",\"贺州\":\"hezhou|10549\",\"防城港\":\"fcg|10539\",\"崇左\":\"chongzuo|10524\",\"桂平\":\"guipingqu|6774\",\n" +
" \"北流\":\"beiliushi|9168\",\"博白\":\"bobaixian|9173\",\"岑溪\":\"cenxi|2119\"\n" +
" }\n" +
" ,\n" +
" \"贵州\":{\n" +
" \"贵阳\":\"gy|2015\",\"遵义\":\"zunyi|7620\",\"黔东南\":\"qdn|9363\",\"黔南\":\"qn|10492\",\"六盘水\":\"lps|10506\",\n" +
" \"毕节\":\"bijie|10564\",\"铜仁\":\"tr|10417\",\"安顺\":\"anshun|7468\",\"黔西南\":\"qxn|10434\",\"仁怀\":\"renhuaishi|7628\",\n" +
" \"清镇\":\"qingzhen|12703\"\n" +
" }\n" +
" ,\n" +
" \"甘肃\":{\n" +
" \"兰州\":\"lz|952\",\"天水\":\"tianshui|8601\",\"白银\":\"by|10304\",\"庆阳\":\"qingyang|10475\",\"平凉\":\"pl|7154\",\n" +
" \"酒泉\":\"jq|10387\",\"张掖\":\"zhangye|10454\",\"武威\":\"wuwei|10448\",\"定西\":\"dx|10322\",\"金昌\":\"jinchang|7428\",\n" +
" \"陇南\":\"ln|10415\",\"临夏\":\"linxia|7112\",\"嘉峪关\":\"jyg|10362\",\"甘南\":\"gn|10343\",\"敦煌\":\"dunhuang|10390\"\n" +
" }\n" +
" ,\n" +
" \"海南\":{\n" +
" \"海口\":\"haikou|2053\",\"三亚\":\"sanya|2422\",\"五指山\":\"wzs|9952\",\"三沙\":\"sansha|13722\",\"琼海\":\"qh|10136\",\n" +
" \"文昌\":\"wenchang|9984\",\"万宁\":\"wanning|10022\",\"屯昌\":\"tunchang|10044\",\"琼中\":\"qiongzhong|10064\",\"陵水\":\"lingshui|10184\",\n" +
" \"东方\":\"df|10250\",\"定安\":\"da|10303\",\"澄迈\":\"cm|10331\",\"保亭\":\"baoting|10367\",\"白沙\":\"baish|10380\",\n" +
" \"儋州\":\"danzhou|10394\"\n" +
" }\n" +
" ,\n" +
" \"河南\":{\n" +
" \"郑州\":\"zz|342\",\"洛阳\":\"luoyang|556\",\"新乡\":\"xx|1016\",\"南阳\":\"ny|592\",\"许昌\":\"xc|977\",\n" +
" \"平顶山\":\"pds|1005\",\"安阳\":\"ay|1096\",\"焦作\":\"jiaozuo|3266\",\"商丘\":\"sq|1029\",\"开封\":\"kaifeng|2342\",\n" +
" \"濮阳\":\"puyang|2346\",\"周口\":\"zk|933\",\"信阳\":\"xy|8694\",\"驻马店\":\"zmd|1067\",\"漯河\":\"luohe|2347\",\n" +
" \"三门峡\":\"smx|9317\",\"鹤壁\":\"hb|2344\",\"济源\":\"jiyuan|9918\",\"明港\":\"mg|8541\",\"鄢陵\":\"yanling|9123\",\n" +
" \"禹州\":\"yuzhou|979\",\"长葛\":\"changge|9344\",\"灵宝\":\"lingbaoshi|9307\",\"杞县\":\"qixianqu|7389\",\"汝州\":\"ruzhou|1010\",\n" +
" \"项城\":\"xiangchengshi|935\",\"偃师\":\"yanshiqu|7121\",\"长垣\":\"changyuan|5936\",\"滑县\":\"huaxian|5405\",\"林州\":\"linzhou|1101\",\n" +
" \"沁阳\":\"qinyang|3268\",\"孟州\":\"mengzhou|3267\",\"温县\":\"wenxian|7312\",\"尉氏\":\"weishixian|7391\",\"兰考\":\"lankaoxian|7393\",\n" +
" \"通许\":\"tongxuxian|7390\",\"新安\":\"lyxinan|11217\",\"伊川\":\"yichuan|11220\",\"孟津\":\"mengjinqu|7122\",\"宜阳\":\"lyyiyang|11219\",\n" +
" \"舞钢\":\"wugang|1011\",\"永城\":\"yongcheng|1032\",\"睢县\":\"suixian|1038\",\"鹿邑\":\"luyi|939\",\"渑池\":\"yingchixian|9322\",\n" +
" \"沈丘\":\"shenqiu|942\",\"太康\":\"taikang|938\",\"商水\":\"shangshui|936\",\"淇县\":\"qixianq|9186\",\"浚县\":\"junxian|9185\",\n" +
" \"范县\":\"fanxian|7285\",\"固始\":\"gushixian|8698\",\"淮滨\":\"huaibinxian|8702\",\"邓州\":\"dengzhou|595\",\"新野\":\"xinye|603\"\n" +
" }\n" +
" ,\n" +
" \"黑龙江\":{\n" +
" \"哈尔滨\":\"hrb|202\",\"大庆\":\"dq|375\",\"齐齐哈尔\":\"qqhr|5853\",\"牡丹江\":\"mdj|3489\",\"绥化\":\"suihua|6718\",\n" +
" \"佳木斯\":\"jms|6776\",\"鸡西\":\"jixi|7289\",\"双鸭山\":\"sys|9837\",\"鹤岗\":\"hegang|9061\",\"黑河\":\"heihe|9862\",\n" +
" \"伊春\":\"yich|9773\",\"七台河\":\"qth|9848\",\"大兴安岭\":\"dxal|9878\",\"安达\":\"shanda|6720\",\"肇东\":\"shzhaodong|6721\",\n" +
" \"肇州\":\"zhaozhou|382\"\n" +
" }\n" +
" ,\n" +
" \"湖北\":{\n" +
" \"武汉\":\"wh|158\",\"宜昌\":\"yc|858\",\"襄阳\":\"xf|891\",\"荆州\":\"jingzhou|3479\",\"十堰\":\"shiyan|2032\",\n" +
" \"黄石\":\"hshi|1734\",\"孝感\":\"xiaogan|3434\",\"黄冈\":\"hg|2299\",\"恩施\":\"es|2302\",\"荆门\":\"jingmen|2296\",\n" +
" \"咸宁\":\"xianning|9617\",\"鄂州\":\"ez|9709\",\"随州\":\"suizhou|9656\",\"潜江\":\"qianjiang|9669\",\"天门\":\"tm|9517\",\n" +
" \"仙桃\":\"xiantao|9736\",\"神农架\":\"snj|9605\",\"宜都\":\"yidou|864\",\"汉川\":\"hanchuan|3439\",\"枣阳\":\"zaoyang|896\",\n" +
" \"武穴\":\"wuxueshi|7362\",\"钟祥\":\"zhongxiangshi|9119\",\"京山\":\"jingshanxian|9117\",\"沙洋\":\"shayangxian|9118\",\"松滋\":\"songzi|3484\",\n" +
" \"广水\":\"guangshuishi|9657\",\"赤壁\":\"chibishi|9623\",\"老河口\":\"laohekou|895\",\"谷城\":\"gucheng|899\",\"宜城\":\"yichengshi|897\",\n" +
" \"南漳\":\"nanzhang|898\",\"云梦\":\"yunmeng|3438\",\"安陆\":\"anlu|3442\",\"大悟\":\"dawu|3437\",\"孝昌\":\"xiaochang|3436\",\n" +
" \"当阳\":\"dangyang|865\",\"枝江\":\"zhijiang|866\",\"嘉鱼\":\"jiayuxian|9624\",\"随县\":\"suixia|9660\"\n" +
" }\n" +
" ,\n" +
" \"湖南\":{\n" +
" \"长沙\":\"cs|414\",\"株洲\":\"zhuzhou|1086\",\"益阳\":\"yiyang|10198\",\"常德\":\"changde|872\",\"衡阳\":\"hy|914\",\n" +
" \"湘潭\":\"xiangtan|2047\",\"岳阳\":\"yy|821\",\"郴州\":\"chenzhou|5695\",\"邵阳\":\"shaoyang|2303\",\"怀化\":\"hh|5756\",\n" +
" \"永州\":\"yongzhou|2307\",\"娄底\":\"ld|9481\",\"湘西\":\"xiangxi|10219\",\"张家界\":\"zjj|6788\",\"醴陵\":\"liling|1091\",\n" +
" \"澧县\":\"lixian|876\",\"桂阳\":\"czguiyang|5699\",\"资兴\":\"zixing|5698\",\"永兴\":\"yongxing|5701\",\"常宁\":\"changningshi|921\",\n" +
" \"祁东\":\"qidongxian|5690\",\"衡东\":\"hengdong|5693\",\"冷水江\":\"lengshuijiangshi|9470\",\"涟源\":\"lianyuanshi|9471\",\"双峰\":\"shuangfengxian|9473\",\n" +
" \"邵阳县\":\"shaoyangxian|6955\",\"邵东\":\"shaodongxian|6954\",\"沅江\":\"yuanjiangs|10201\",\"南县\":\"nanxian|10202\",\"祁阳\":\"qiyang|8532\",\n" +
" \"湘阴\":\"xiangyin|828\",\"华容\":\"huarong|830\",\"慈利\":\"cilixian|6791\",\"攸县\":\"zzyouxian|1095\"\n" +
" }\n" +
" ,\n" +
" \"河北\":{\n" +
" \"石家庄\":\"sjz|241\",\"保定\":\"bd|424\",\"唐山\":\"ts|276\",\"廊坊\":\"lf|772\",\"邯郸\":\"hd|572\",\n" +
" \"秦皇岛\":\"qhd|1078\",\"沧州\":\"cangzhou|652\",\"邢台\":\"xt|751\",\"衡水\":\"hs|993\",\"张家口\":\"zjk|3328\",\n" +
" \"承德\":\"chengde|6760\",\"定州\":\"dingzhou|8398\",\"馆陶\":\"gt|8706\",\"张北\":\"zhangbei|11201\",\"赵县\":\"zx|9048\",\n" +
" \"正定\":\"zd|3198\",\"迁安市\":\"qianan|284\",\"任丘\":\"renqiu|656\",\"三河\":\"sanhe|776\",\"武安\":\"wuan|577\",\n" +
" \"雄安新区\":\"xionganxinqu|111234\",\"燕郊\":\"lfyanjiao|12804\",\"涿州\":\"zhuozhou|428\",\"河间\":\"hejian|658\",\"黄骅\":\"huanghua|657\",\n" +
" \"沧县\":\"cangxian|659\",\"磁县\":\"cixian|591\",\"涉县\":\"shexian|14059\",\"霸州\":\"bazhou|775\",\"香河\":\"xianghe|5395\",\n" +
" \"固安\":\"lfguan|12803\",\"遵化市\":\"zunhua|283\",\"迁西\":\"qianxixian|7061\",\"玉田\":\"yutianxian|7060\",\"滦南\":\"luannanxian|7066\",\n" +
" \"沙河\":\"shaheshi|755\"\n" +
" }\n" +
" ,\n" +
" \"江苏\":{\n" +
" \"苏州\":\"su|5\",\"南京\":\"nj|172\",\"无锡\":\"wx|93\",\"常州\":\"cz|463\",\"徐州\":\"xz|471\",\n" +
" \"南通\":\"nt|394\",\"扬州\":\"yz|637\",\"盐城\":\"yancheng|613\",\"淮安\":\"ha|968\",\"连云港\":\"lyg|2049\",\n" +
" \"泰州\":\"taizhou|693\",\"宿迁\":\"suqian|2350\",\"镇江\":\"zj|645\",\"沭阳\":\"shuyang|5772\",\"大丰\":\"dafeng|11279\",\n" +
" \"如皋\":\"rugao|397\",\"启东\":\"qidong|400\",\"溧阳\":\"liyang|469\",\"海门\":\"haimen|399\",\"东海\":\"donghai|2147\",\n" +
" \"扬中\":\"yangzhong|649\",\"兴化\":\"xinghuashi|699\",\"新沂\":\"xinyishi|478\",\"泰兴\":\"taixing|696\",\"如东\":\"rudong|402\",\n" +
" \"邳州\":\"pizhou|477\",\"沛县\":\"xzpeixian|11349\",\"靖江\":\"jingjiang|698\",\"建湖\":\"jianhu|618\",\"海安\":\"haian|401\",\n" +
" \"东台\":\"dongtai|615\",\"丹阳\":\"danyang|648\",\"宝应县\":\"baoyingx|14451\",\"灌南\":\"guannan|2150\",\"灌云\":\"guanyun|2148\",\n" +
" \"姜堰\":\"jiangyan|697\",\"金坛\":\"jintan|468\",\"昆山\":\"szkunshan|16\",\"泗洪\":\"sihong|5958\",\"泗阳\":\"siyang|5959\",\n" +
" \"句容\":\"jurong|650\",\"射阳\":\"sheyang|621\",\"阜宁\":\"funingxian|620\",\"响水\":\"xiangshui|619\",\"盱眙\":\"xuyi|976\",\n" +
" \"金湖\":\"jinhu|975\",\"江阴\":\"jiangyins|34984\"\n" +
" }\n" +
" ,\n" +
" \"江西\":{\n" +
" \"南昌\":\"nc|669\",\"赣州\":\"ganzhou|2363\",\"九江\":\"jj|2247\",\"宜春\":\"yichun|5709\",\"吉安\":\"ja|2364\",\n" +
" \"上饶\":\"sr|10120\",\"萍乡\":\"px|2248\",\"抚州\":\"fuzhou|10134\",\"景德镇\":\"jdz|2360\",\"新余\":\"xinyu|10115\",\n" +
" \"鹰潭\":\"yingtan|3209\",\"永新\":\"yxx|11077\",\"乐平\":\"lepingshi|9048\",\"进贤\":\"jinxian|677\",\"分宜\":\"fenyi|10118\",\n" +
" \"丰城\":\"fengchengshi|5711\",\"樟树\":\"zhangshu|5713\",\"高安\":\"gaoan|5712\",\"余江\":\"yujiang|3210\",\"南城\":\"nanchengx|10137\",\n" +
" \"浮梁\":\"fuliangxian|9071\"\n" +
" }\n" +
" ,\n" +
" \"吉林\":{\n" +
" \"长春\":\"cc|319\",\"吉林\":\"jl|700\",\"四平\":\"sp|10171\",\"延边\":\"yanbian|3184\",\"松原\":\"songyuan|2315\",\n" +
" \"白城\":\"bc|5918\",\"通化\":\"th|10159\",\"白山\":\"baishan|10179\",\"辽源\":\"liaoyuan|2501\",\"公主岭\":\"gongzhuling|10171\",\n" +
" \"梅河口\":\"meihekou|10162\",\"扶余\":\"fuyuxian|9085\",\"长岭\":\"changlingxian|9084\",\"桦甸\":\"huadian|706\",\"磐石\":\"panshi|708\",\n" +
" \"梨树县\":\"lishu|10176\"\n" +
" }\n" +
" ,\n" +
" \"辽宁\":{\n" +
" \"沈阳\":\"sy|188\",\"大连\":\"dl|147\",\"鞍山\":\"as|523\",\"锦州\":\"jinzhou|2354\",\"抚顺\":\"fushun|5722\",\n" +
" \"营口\":\"yk|5898\",\"盘锦\":\"pj|2041\",\"朝阳\":\"cy|10106\",\"丹东\":\"dandong|3445\",\"辽阳\":\"liaoyang|2038\",\n" +
" \"本溪\":\"benxi|5845\",\"葫芦岛\":\"hld|10088\",\"铁岭\":\"tl|6729\",\"阜新\":\"fx|10097\",\"庄河\":\"pld|3306\",\n" +
" \"瓦房店\":\"wfd|3279\",\"灯塔\":\"dengta|2071\",\"凤城\":\"fengcheng|3450\",\"北票\":\"beipiao|10109\",\"开原\":\"kaiyuan|6733\"\n" +
" }\n" +
" ,\n" +
" \"宁夏\":{\n" +
" \"银川\":\"yinchuan|2054\",\"吴忠\":\"wuzhong|9962\",\"石嘴山\":\"szs|9971\",\"中卫\":\"zw|9951\",\"固原\":\"guyuan|2421\"\n" +
" }\n" +
" ,\n" +
" \"内蒙古\":{\n" +
" \"呼和浩特\":\"hu|811\",\"包头\":\"bt|801\",\"赤峰\":\"chifeng|6700\",\"鄂尔多斯\":\"erds|2037\",\"通辽\":\"tongliao|10015\",\n" +
" \"呼伦贝尔\":\"hlbe|10039\",\"巴彦淖尔市\":\"bycem|10070\",\"乌兰察布\":\"wlcb|9993\",\"锡林郭勒\":\"xl|2408\",\"兴安盟\":\"xam|9976\",\n" +
" \"乌海\":\"wuhai|2404\",\"阿拉善盟\":\"alsm|10083\",\"海拉尔\":\"hlr|2043\"\n" +
" }\n" +
" ,\n" +
" \"青海\":{\n" +
" \"西宁\":\"xn|2052\",\"海西\":\"hx|9902\",\"海北\":\"haibei|9917\",\"果洛\":\"guoluo|9936\",\"海东\":\"haidong|9909\",\n" +
" \"黄南\":\"huangnan|9896\",\"玉树\":\"ys|9888\",\"海南\":\"hainan|10574\",\"格尔木\":\"geermushi|9904\"\n" +
" }\n" +
" ,\n" +
" \"山东\":{\n" +
" \"青岛\":\"qd|122\",\"济南\":\"jn|265\",\"烟台\":\"yt|228\",\"潍坊\":\"wf|362\",\"临沂\":\"linyi|505\",\n" +
" \"淄博\":\"zb|385\",\"济宁\":\"jining|450\",\"泰安\":\"ta|686\",\"聊城\":\"lc|882\",\"威海\":\"weihai|518\",\n" +
" \"枣庄\":\"zaozhuang|961\",\"德州\":\"dz|728\",\"日照\":\"rizhao|3177\",\"东营\":\"dy|623\",\"菏泽\":\"heze|5632\",\n" +
" \"滨州\":\"bz|944\",\"莱芜\":\"lw|2292\",\"章丘\":\"zhangqiu|8680\",\"垦利\":\"kl|11313\",\"诸城\":\"zc|9146\",\n" +
" \"寿光\":\"shouguang|369\",\"龙口\":\"longkou|233\",\"曹县\":\"caoxian|5638\",\"单县\":\"shanxian|5636\",\"肥城\":\"feicheng|690\",\n" +
" \"高密\":\"gaomi|371\",\"广饶\":\"guangrao|627\",\"桓台\":\"huantaixian|7335\",\"莒县\":\"juxian|3180\",\"莱州\":\"laizhou|235\",\n" +
" \"蓬莱\":\"penglai|237\",\"青州\":\"qingzhou|367\",\"荣成\":\"rongcheng|522\",\"乳山\":\"rushan|520\",\"滕州\":\"tengzhou|967\",\n" +
" \"新泰\":\"xintai|689\",\"招远\":\"zhaoyuan|3325\",\"邹城\":\"zoucheng|455\",\"邹平\":\"zouping|946\",\"临清\":\"linqing|884\",\n" +
" \"茌平\":\"chiping|887\",\"郓城\":\"hzyc|5637\",\"博兴\":\"boxing|949\",\"东明\":\"dongming|5641\",\"巨野\":\"juye|5640\",\n" +
" \"无棣\":\"wudi|951\",\"齐河\":\"qihe|734\",\"微山\":\"weishan|459\",\"禹城\":\"yuchengshi|731\",\"临邑\":\"linyixianq|739\",\n" +
" \"乐陵\":\"leling|730\",\"莱阳\":\"laiyang|234\",\"宁津\":\"ningjin|733\",\"高唐\":\"gaotang|885\",\"莘县\":\"shenxian|888\",\n" +
" \"阳谷\":\"yanggu|886\",\"冠县\":\"guanxian|890\",\"平邑\":\"pingyi|514\",\"郯城\":\"tancheng|510\",\"沂源\":\"yiyuanxian|7334\",\n" +
" \"汶上\":\"wenshang|460\",\"梁山\":\"liangshanx|462\",\"利津\":\"lijin|628\",\"沂南\":\"yinanxian|7301\",\"栖霞\":\"qixia|238\",\n" +
" \"宁阳\":\"ningyang|691\",\"东平\":\"dongping|692\",\"昌邑\":\"changyishi|372\",\"安丘\":\"anqiu|370\",\"昌乐\":\"changle|373\",\n" +
" \"临朐\":\"linqu|374\",\"鄄城\":\"juancheng|5635\"\n" +
" }\n" +
" ,\n" +
" \"山西\":{\n" +
" \"太原\":\"ty|740\",\"临汾\":\"linfen|5669\",\"大同\":\"dt|6964\",\"运城\":\"yuncheng|5653\",\"晋中\":\"jz|8854\",\n" +
" \"长治\":\"changzhi|6921\",\"晋城\":\"jincheng|3350\",\"阳泉\":\"yq|8760\",\"吕梁\":\"lvliang|3222\",\"忻州\":\"xinzhou|3453\",\n" +
" \"朔州\":\"shuozhou|9871\",\"临猗\":\"linyixian|9193\",\"清徐\":\"qingxu|10908\",\"柳林\":\"liulin|3225\",\"高平\":\"gaoping|3354\",\n" +
" \"泽州\":\"zezhou|3353\",\"襄垣\":\"xiangyuanxian|6928\",\"孝义\":\"xiaoyi|3227\"\n" +
" }\n" +
" ,\n" +
" \"陕西\":{\n" +
" \"西安\":\"xa|483\",\"咸阳\":\"xianyang|7453\",\"宝鸡\":\"baoji|2044\",\"渭南\":\"wn|5733\",\"汉中\":\"hanzhong|3163\",\n" +
" \"榆林\":\"yl|5942\",\"延安\":\"yanan|8973\",\"安康\":\"ankang|3157\",\"商洛\":\"sl|9854\",\"铜川\":\"tc|9832\",\n" +
" \"神木\":\"shenmu|5944\",\"韩城\":\"hancheng|5735\",\"府谷\":\"fugu|5945\",\"靖边\":\"jingbian|5947\",\"定边\":\"dingbian|5948\"\n" +
" }\n" +
" ,\n" +
" \"四川\":{\n" +
" \"成都\":\"cd|102\",\"绵阳\":\"mianyang|1057\",\"德阳\":\"deyang|2373\",\"南充\":\"nanchong|2378\",\"宜宾\":\"yb|2380\",\n" +
" \"自贡\":\"zg|6745\",\"乐山\":\"ls|3237\",\"泸州\":\"luzhou|2372\",\"达州\":\"dazhou|9799\",\"内江\":\"scnj|5928\",\n" +
" \"遂宁\":\"suining|9688\",\"攀枝花\":\"panzhihua|2371\",\"眉山\":\"ms|9704\",\"广安\":\"ga|2381\",\"资阳\":\"zy|6803\",\n" +
" \"凉山\":\"liangshan|9717\",\"广元\":\"guangyuan|9749\",\"雅安\":\"ya|9687\",\"巴中\":\"bazhong|9811\",\"阿坝\":\"ab|9817\",\n" +
" \"甘孜\":\"ganzi|9764\",\"安岳\":\"anyuexian|6806\",\"广汉\":\"guanghanshi|8719\",\"简阳\":\"jianyangshi|6805\",\"仁寿\":\"renshouxian|9706\",\n" +
" \"射洪\":\"shehongxian|9694\",\"大竹\":\"dazu|9806\",\"宣汉\":\"xuanhan|9804\",\"渠县\":\"qux|9807\",\"长宁\":\"changningx|7148\"\n" +
" }\n" +
" ,\n" +
" \"新疆\":{\n" +
" \"乌鲁木齐\":\"xj|984\",\"昌吉\":\"changji|8582\",\"巴音郭楞\":\"bygl|9530\",\"伊犁\":\"yili|9472\",\"阿克苏\":\"aks|9499\",\n" +
" \"喀什\":\"ks|9326\",\"哈密\":\"hami|7452\",\"克拉玛依\":\"klmy|2042\",\"博尔塔拉\":\"betl|9529\",\"吐鲁番\":\"tlf|9475\",\n" +
" \"和田\":\"ht|9489\",\"石河子\":\"shz|9551\",\"克孜勒苏\":\"kzls|9519\",\"阿拉尔\":\"ale|9539\",\"五家渠\":\"wjq|9562\",\n" +
" \"图木舒克\":\"tmsk|9559\",\"库尔勒\":\"kel|7168\",\"阿勒泰\":\"alt|18837\",\"塔城\":\"tac|18845\"\n" +
" }\n" +
" ,\n" +
" \"西藏\":{\n" +
" \"拉萨\":\"lasa|2055\",\"日喀则\":\"rkz|9615\",\"山南\":\"sn|9576\",\"林芝\":\"linzhi|9646\",\"昌都\":\"changdu|9648\",\n" +
" \"那曲\":\"nq|9618\",\"阿里\":\"al|9678\",\"日土\":\"rituxian|9682\",\"改则\":\"gaizexian|9684\"\n" +
" }\n" +
" ,\n" +
" \"云南\":{\n" +
" \"昆明\":\"km|541\",\"曲靖\":\"qj|2389\",\"大理\":\"dali|2398\",\"红河\":\"honghe|2394\",\"玉溪\":\"yx|2040\",\n" +
" \"丽江\":\"lj|2392\",\"文山\":\"ws|2395\",\"楚雄\":\"cx|2393\",\"西双版纳\":\"bn|2397\",\"昭通\":\"zt|9409\",\n" +
" \"德宏\":\"dh|9437\",\"普洱\":\"pe|9444\",\"保山\":\"bs|2390\",\"临沧\":\"lincang|9422\",\"迪庆\":\"diqing|9432\",\n" +
" \"怒江\":\"nujiang|9462\",\"弥勒\":\"milexian|8892\",\"安宁\":\"anningshi|547\",\"宣威\":\"xuanwushi|7533\"\n" +
" }\n" +
" ,\n" +
" \"浙江\":{\n" +
" \"杭州\":\"hz|79\",\"宁波\":\"nb|135\",\"温州\":\"wz|330\",\"金华\":\"jh|531\",\"嘉兴\":\"jx|497\",\n" +
" \"台州\":\"tz|403\",\"绍兴\":\"sx|355\",\"湖州\":\"huzhou|831\",\"丽水\":\"lishui|7921\",\"衢州\":\"quzhou|6793\",\n" +
" \"舟山\":\"zhoushan|8481\",\"乐清\":\"yueqingcity|13950\",\"瑞安\":\"ruiancity|13951\",\"义乌\":\"yiwu|12291\",\"余姚\":\"yuyao|5333\",\n" +
" \"诸暨\":\"zhuji|357\",\"象山\":\"xiangshanxian|6738\",\"温岭\":\"wenling|408\",\"桐乡\":\"tongxiang|502\",\"慈溪\":\"cixi|5334\",\n" +
" \"长兴\":\"changxing|834\",\"嘉善\":\"jiashanx|14357\",\"海宁\":\"haining|500\",\"德清\":\"deqing|835\",\"东阳\":\"dongyang|536\",\n" +
" \"安吉\":\"anji|836\",\"苍南\":\"cangnanxian|7579\",\"临海\":\"linhai|407\",\"永康\":\"yongkang|537\",\"玉环\":\"yuhuan|409\",\n" +
" \"平湖\":\"pinghushi|501\",\"海盐\":\"haiyan|504\",\"武义县\":\"wuyix|14528\",\"嵊州\":\"shengzhou|359\",\"新昌\":\"xinchang|361\",\n" +
" \"江山\":\"jiangshanshi|6796\",\"平阳\":\"pingyangxian|7575\"\n" +
" }\n" +
" ,\n" +
" \"其他\":{\n" +
" \"中国香港\":\"hk|2050\",\"中国澳门\":\"am|9399\",\"中国台湾\":\"tw|2051\"\n" +
" }\n" +
" ,\n" +
" \"海外\":{\n" +
" \"洛杉矶\":\"gllosangeles\",\"旧金山\":\"glsanfrancisco\",\"纽约\":\"glnewyork\",\"多伦多\":\"gltoronto\",\"温哥华\":\"glvancouver\",\n" +
" \"伦敦\":\"glgreaterlondon\",\"莫斯科\":\"glmoscow\",\"首尔\":\"glseoul\",\"东京\":\"gltokyo\",\"新加坡\":\"glsingapore\",\n" +
" \"曼谷\":\"glbangkok\",\"清迈\":\"glchiangmai\",\"迪拜\":\"gldubai\",\"奥克兰\":\"glauckland\",\"悉尼\":\"glsydney\",\n" +
" \"墨尔本\":\"glmelbourne\",\"其他海外城市\":\"city\"\n" +
" }\n" +
" }";
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static Set citySet = CityUtils.listCitySet()
static void main(String[] args) {
JSONObject jsonObject = JSONObject.parseObject(cityList)
Set<String> provinceSet = jsonObject.keySet()
List<DBObject> ret = new ArrayList<>()
for (String province: provinceSet) {
JSONObject provinceInfo = jsonObject[province]
for (String city : provinceInfo.keySet()) {
// 屏蔽县
if (citySet.contains(city)) {
def city_short = provinceInfo[city].split("\\|")[0]
DBObject dbObject = new BasicDBObject()
dbObject.put("province", province)
dbObject.put("city", city)
dbObject.put("city_short", city_short)
dbObject.put("c_date", new Date())
dbObject.put("m_date", new Date())
dbObject.put("root", "58.com")
ret.add(dbObject)
}
}
}
cityConn.insert(ret)
print("58城市数据写入成功")
}
}
@@ -0,0 +1,67 @@
package com.fangjia.script.temp.shenjian.crawl
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.time.DateUtils
/**
* @author shenjian
* @since 2021/4/30
*/
class Comm58 {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static final def source = "58同城"
static final def root = "58.com"
static void main(String[] args) {
Date c_date = new Date()
DBObject query = new BasicDBObject()
query.put("source",source)
query.put("city","福州")
query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd")));
DBObject filter = new BasicDBObject()
filter.put("district_id", 1)
filter.put("city", 1)
filter.put("region", 1)
Set<String> set = new HashSet<>()
List doc = new ArrayList()
collection.find(query, filter).batchSize(10000).each { obj ->
try {
def url = obj.district_id
def cityShort = "fz"
String[] info = url.split("/")
def id = info[info.size() - 1]
if (!set.contains(id)) {
DBObject dbObject = new BasicDBObject()
dbObject.put("city", obj.city)
dbObject.put("region", obj.region)
dbObject.put("city_short", cityShort)
dbObject.put("district_id", id)
dbObject.put("root", root)
dbObject.put("c_date", c_date)
dbObject.put("m_date", c_date)
if (obj.region != null) {
set.add(id)
doc.add(dbObject)
}
if (doc.size() == 5000) {
districtSeeds.insert(doc)
doc.clear()
}
}
} catch(Exception ex) {
println("解析异常:" + obj)
}
}
districtSeeds.insert(doc)
doc.clear()
print("操作完成")
}
}
@@ -0,0 +1,65 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.crawl.InvokeHttpRequest
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
/**
* 统计58小区总量
*
* @author shenjian
* @since 2021/5/26
*/
class Count58Seaweed {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static void main(String[] args) {
crawlSeaweedCount()
println(count())
}
static crawlSeaweedCount() {
cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)),
new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1))
.batchSize(100).each { obj ->
def id = obj._id
def city_short = obj.city_short
final def URL = "https://" + city_short + ".58.com/xiaoqu/"
try {
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res) {
Document doc = Jsoup.parse(res);
String str = JQueryUtils.selectText(".total-info", doc)
// 117.60.236.41
if (str) {
str = str.replace("共找到", "").replace("个小区", "").trim()
int count = Integer.parseInt(str)
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count)))
println("当前处理城市" + obj.city)
}
}
} catch(Exception ex) {
ex.printStackTrace()
}
}
}
static count() {
int number = 0;
cityConn.find(new BasicDBObject("root", "58.com"), new BasicDBObject("district_count", 1))
.batchSize(100).each { obj ->
if (obj.district_count) {
number += obj.district_count
}
}
return number
}
}
@@ -0,0 +1,73 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.util.logging.Slf4j
import org.springframework.util.StopWatch
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 58小区详情爬取
*
* @author shenjian* @since 2021/5/26
*/
@Slf4j
class District58DetailCrawl {
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static void main(String[] args) {
AtomicLong count = new AtomicLong()
CountDownLatch latch = new CountDownLatch(100000)
districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
.batchSize(1000).limit(100000).each { objTemp ->
def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
try {
final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/"
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res && res.code == 200) {
String content = res.content
if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) {
if (content.indexOf("您所访问的页面不存在") != -1) {
districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在")))
} else {
JSON json = new JSONObject()
json.put("project", "district_detail_58")
json.put("url", URL)
JSON result = new JSONObject()
result.put("content", (String) content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
districtSeedConn.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
log.info("已抓取 {}", obj.district_id)
}
}
}
} catch (Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
}
}
@@ -0,0 +1,44 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 58有效种子梳理
*
* @author shenjian
* @since 2021/5/14
*/
class Seaweed58 {
static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static void main(String[] args) {
List<String> list = cityMapConn.distinct("fangjia_city")
Set<String> citySet = new HashSet<>(list);
DBObject query = new BasicDBObject()
query.put("root", "58.com")
DBObject filter = new BasicDBObject()
filter.put("city", "1")
filter.put("_id", "1")
def delIds = []
districtSeedsConn.find(query, filter).batchSize(20000).each {obj ->
if (!citySet.contains(obj.city)) {
delIds.add(obj._id)
if (delIds.size() == 10000) {
districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds)))
delIds = []
}
}
}
districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds)))
delIds = []
print("58种子清理成功")
}
}
@@ -0,0 +1,56 @@
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.time.DateUtils
/**
* 房天下小区种子生成
*/
class CommFang {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static void main(String[] args) {
Date c_date = new Date()
DBObject query = new BasicDBObject()
query.put("source","搜房")
query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd")));
DBObject filter = new BasicDBObject()
filter.put("district_id", 1)
filter.put("city", 1)
filter.put("region", 1)
Set<String> set = new HashSet<>()
List doc = new ArrayList()
collection.find(query, filter).batchSize(1000).each { obj ->
def url = obj.district_id
def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf("."))
def id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf("."))
if (!set.contains(id) && StringUtils.isNumeric(id)) {
DBObject dbObject = new BasicDBObject()
dbObject.put("city", obj.city)
dbObject.put("region", obj.region)
dbObject.put("city_short", cityShort)
dbObject.put("district_id", id)
dbObject.put("root", "fang.com")
dbObject.put("c_date", c_date)
dbObject.put("m_date", c_date)
if (obj.region != null) {
set.add(id)
doc.add(dbObject)
}
if (doc.size() == 5000) {
districtSeeds.insert(doc)
doc.clear()
}
}
}
districtSeeds.insert(doc)
doc.clear()
print("操作完成")
}
}
@@ -0,0 +1,57 @@
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.time.DateUtils
/**
* 贝壳找房小区种子生成
*/
class CommKe {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static final def source = "贝壳找房"
static final def root = "ke.com"
static void main(String[] args) {
Date c_date = new Date()
DBObject query = new BasicDBObject()
query.put("source",source)
query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd")));
DBObject filter = new BasicDBObject()
filter.put("url", 1)
filter.put("district_id", 1)
filter.put("city", 1)
filter.put("region", 1)
Set<String> set = new HashSet<>()
List doc = new ArrayList()
collection.find(query, filter).batchSize(1000).each { obj ->
def url = obj.url
def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf("."))
def id = obj.district_id
if (!set.contains(id) && StringUtils.isNumeric(id)) {
DBObject dbObject = new BasicDBObject()
dbObject.put("city", obj.city)
dbObject.put("region", obj.region)
dbObject.put("city_short", cityShort)
dbObject.put("district_id", id)
dbObject.put("root", root)
dbObject.put("c_date", c_date)
dbObject.put("m_date", c_date)
set.add(id)
doc.add(dbObject)
if (doc.size() == 5000) {
districtSeeds.insert(doc)
doc.clear()
}
}
}
districtSeeds.insert(doc)
doc.clear()
print("操作完成")
}
}
@@ -0,0 +1,33 @@
package com.fangjia.script.temp.shenjian.crawl
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 种子数据备份
*
* @author shenjian
* @since 2021/5/14
*/
class SeedBak {
static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static DBCollection aimConn = MongoUtils.getCollectionByName("seeddb_dev", "district_seeds", "192.168.0.235", 27018)
static void main(String[] args) {
List<DBObject> list = new ArrayList<>()
districtSeedsConn.find().batchSize(20000).each {obj ->
list.add(obj)
if (list.size() == 20000) {
aimConn.insert(list)
list.clear()
}
}
aimConn.insert(list)
list.clear()
println("种子备份成功")
}
}
@@ -0,0 +1,75 @@
package com.fangjia.script.temp.shenjian.crawl.anjuke
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.util.logging.Slf4j
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 安居客小区详情抓取
*
* @author shenjian
* @since 2021/5/28
*/
@Slf4j
class AnjukeDistrictDetailCrawl {
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static void main(String[] args) {
AtomicLong count = new AtomicLong()
CountDownLatch latch = new CountDownLatch(100000)
districtSeedConn.find(new BasicDBObject("root", "anjuke.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
.batchSize(1000).limit(1000).each { objTemp ->
def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
try {
final def URL = "https://" + obj.city_short + ".anjuke.com/community/view/" + obj.district_id + "#/"
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res && res.code == 200) {
String content = res.content
if (content.indexOf("请拖动滑块进行验证") == -1 && content.indexOf("访问验证-安居客") == -1) {
if (content.indexOf("您所访问的页面不存在") != -1) {
districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在")))
} else {
JSON json = new JSONObject()
json.put("project", "district_detail_anjuke")
json.put("url", URL)
JSON result = new JSONObject()
result.put("content", (String) content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
districtSeedConn.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
log.info("已抓取 {}", obj.district_id)
}
}
} else if (res && res.code == 508) {
log.info("508 {}", obj.district_id)
}
} catch (Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
}
}
@@ -0,0 +1,56 @@
package com.fangjia.script.temp.shenjian.crawl.ke
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.temp.shenjian.util.CityUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
/**
* 贝壳找房城市种子生成
*
* @author shenjian
* @since 2021/6/2
*/
class CityFang {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static Set citySet = CityUtils.listCitySet()
static void main(String[] args) {
// 房天下较为特殊,直接从url取乱码,需要改页面所有非UTF-8编码为UTF-8
Document doc = Jsoup.parse(new File("F:\\数据\\爬虫\\fang_city.html"), "UTF-8");
Elements elements = JQueryUtils.select(".outCont li", doc)
List<DBObject> ret = new ArrayList<>()
for (Element element : elements) {
String province = JQueryUtils.selectText("strong", element)
Elements cityElements = JQueryUtils.select("a", element)
for (Element cityElement: cityElements) {
String city = cityElement.text()
if ("直辖市".equals(province)) {
province = city
}
if (citySet.contains(city)) {
String url = cityElement.attr("href")
String cityShort = StringUtils.substringBetween(url,"//", ".esf.fang.com")
DBObject dbObject = new BasicDBObject()
dbObject.put("province", province)
dbObject.put("city", city)
dbObject.put("city_short", cityShort)
dbObject.put("c_date", new Date())
dbObject.put("m_date", new Date())
dbObject.put("root", "fang.com")
ret.add(dbObject)
}
}
}
cityConn.insert(ret)
println("房天下城市数据写入成功")
}
}
@@ -0,0 +1,57 @@
package com.fangjia.script.temp.shenjian.crawl.ke
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.temp.shenjian.util.CityUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
/**
* 贝壳找房城市种子生成
*
* @author shenjian
* @since 2021/6/2
*/
class CityKe {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static Set citySet = CityUtils.listCitySet()
static void main(String[] args) {
final def URL = "https://www.ke.com/city/"
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res) {
Document doc = Jsoup.parse(res);
Elements elements = JQueryUtils.select(".city_list_ul .city_list_li", doc)
List<DBObject> ret = new ArrayList<>()
for (Element element : elements) {
String province = JQueryUtils.selectText(".city_province .city_list_tit", element)
Elements cityElements = JQueryUtils.select(".CLICKDATA a", element)
for (Element cityElement: cityElements) {
String city = cityElement.text()
if (citySet.contains(city)) {
String url = cityElement.attr("href")
String cityShort = StringUtils.substringBetween(url,"//", ".ke.com")
DBObject dbObject = new BasicDBObject()
dbObject.put("province", province)
dbObject.put("city", city)
dbObject.put("city_short", cityShort)
dbObject.put("c_date", new Date())
dbObject.put("m_date", new Date())
dbObject.put("root", "ke.com")
ret.add(dbObject)
}
}
}
cityConn.insert(ret)
println("贝壳找房城市数据写入成功")
}
}
}
@@ -0,0 +1,100 @@
package com.fangjia.script.temp.shenjian.crawl.ke
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 贝壳各城市成交总量抓取
*
* @author shenjian
* @since 2021/6/2
*/
class CountTradeKe {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
//crawlTradeCount()
//println(count())
//1896919 1760177
findUnNormalCount()
}
static crawlTradeCount() {
cityConn.find(new BasicDBObject("root", "ke.com"),
new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1))
.batchSize(100).each { objTemp ->
final def obj = objTemp
executorService.execute(new Runnable() {
@Override
void run() {
def id = obj._id
def city_short = obj.city_short
final def URL = "https://" + city_short + ".ke.com/chengjiao/"
try {
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res) {
Document doc = Jsoup.parse(res);
boolean existTrade = JQueryUtils.selectText(".resultDes .total", doc).contains("成交房源")
if (existTrade) {
String str = JQueryUtils.selectText(".resultDes .total span", doc)
int count = Integer.parseInt(str)
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", count)))
} else {
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", 0)))
}
println("当前处理城市" + obj.city)
}
} catch(Exception ex) {
ex.printStackTrace()
}
}
})
}
executorService.shutdown()
}
/**
* 找出异常抓取的城市
*/
static findUnNormalCount() {
cityConn.find(new BasicDBObject("root", "ke.com"),
new BasicDBObject("_id", 1).append("city", 1).append("city_short", 1).append("trade_count", 1))
.batchSize(100).each { obj ->
Long realTradeCount = obj.trade_count
if (realTradeCount > 0) {
Long crawledCount = dealPriceConn.count(new BasicDBObject("city", obj.city).append("source", "贝壳找房"))
if (realTradeCount > crawledCount) {
println(obj.city + " " + obj.city_short + "抓取异常:抓取为" + crawledCount + " 实际为" + realTradeCount + " 少抓" + (realTradeCount - crawledCount))
}
}
}
}
/**
* 统计实际抓取量
*
* @return
*/
static count() {
int number = 0;
cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1))
.batchSize(100).each { obj ->
if (obj.trade_count) {
number += obj.trade_count
}
}
return number
}
}
@@ -0,0 +1,24 @@
package com.fangjia.script.temp.shenjian.dingding
import com.fangjia.script.util.DateUtils
/**
* ¶¤¶¤½Ó¿Úµ÷Óù¤¾ßÀà
*
* @author shenjian
* @since 2021/5/8
*/
class DingTalkUtils {
static main(String[] args) {
Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ;
String startTime = DateUtils.addMonth(now, -1) + " 00:00:00"
String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss")
String endTime = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(now), "yyyy-MM-dd") + " 23:59:59"
ScoreExport.export(startTime, monthStartTime, endTime)
ProjectExport.export(startTime, monthStartTime, endTime)
TaskExport.export(startTime, monthStartTime, endTime)
}
}
@@ -0,0 +1,110 @@
package com.fangjia.script.temp.shenjian.dingding
import com.fangjia.script.statistics.ApiUtil
import com.fangjia.script.statistics.Project
import com.fangjia.script.statistics.ProjectLabel
import com.fangjia.script.statistics.Task
import com.fangjia.script.statistics.TaskDetail
import org.apache.commons.collections.CollectionUtils
/**
* 项目导出
*
* @author shenjian
* @since 2021/5/8
*/
class ProjectExport {
static final String time = "2021年4月1日-2021年4月30日";
static final String startTime = "2021-03-01 00:00:00"
static final String monthStartTime = "2021-04-01 00:00:00"
static final String endTime = "2021-04-30 23:59:59"
static main(args) {
export(startTime, monthStartTime, endTime)
}
static export(def startTime, def monthStartTime, def endTime) {
List<List<Integer>> mergeData = new ArrayList<>()
def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"];
def result = []
List<Project> projects = ApiUtil.listProject();
for (Project project : projects) {
def ret = []
List<ProjectLabel> labels = ApiUtil.listProjectLabel(project.getId())
ret[1] = project.getName()
for (ProjectLabel label : labels) {
if ("商务负责人" == label.getGroups().getName()) {
if (ret[2] == null) {
ret[2] = label.getName()
} else {
ret[2] = ret[2] + "、" + label.getName()
}
} else if ("项目类别" == label.getGroups().getName()) {
ret[0] = label.getName()
}
}
// 获取项目总工时
int limitStart = 0
List<Task> tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50)
List<Task> tempTasks = tasks;
while (tasks != null && tempTasks != null && tempTasks.size() == 50) {
limitStart += 50
tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50)
if (CollectionUtils.isNotEmpty(tempTasks)) {
tasks.addAll(tempTasks)
}
}
Double totalTime = 0D;
if (CollectionUtils.isNotEmpty(tasks)) {
for (Task task: tasks) {
// 获取计划时间在当月的任务
TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId())
if (taskDetail == null) {
print("当前任务taskDetail为空:" + task)
Thread.sleep(60 * 1000)
taskDetail = ApiUtil.getTaskDetail(task.getId())
}
if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime)
if (taskDetail.getFlowConten() != null) {
totalTime += Double.valueOf(taskDetail.getFlowConten())
}
}
}
ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, totalTime)
result << ret
}
def sortedResult = []
sortedResult << ['项目类别', '项目名称', '项目负责人', '总工时(天)']
for (def label : sort) {
List<Integer> mergeRowCel = new ArrayList<>()
mergeRowCel.add(sortedResult.size())
for (def ret : result) {
if (ret[0] == label) {
sortedResult << ret
}
}
mergeRowCel.add(sortedResult.size() - 1)
mergeRowCel.add(0)
mergeRowCel.add(0)
mergeData.add(mergeRowCel)
}
com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计", mergeData);
}
}
@@ -0,0 +1,154 @@
package com.fangjia.script.temp.shenjian.dingding
import com.fangjia.script.statistics.ApiUtil
import com.fangjia.script.statistics.Project
import com.fangjia.script.statistics.ProjectLabel
import com.fangjia.script.statistics.Task
import com.fangjia.script.statistics.TaskDetail
import org.apache.commons.collections.CollectionUtils
/**
* 绩效导出
*
* @author shenjian
* @since 2021/5/10
*/
public class ScoreExport {
static final String time = "2021年4月1日-2021年4月30日";
static final String startTime = "2021-03-01 00:00:00"
static final String monthStartTime = "2021-04-01 00:00:00"
static final String endTime = "2021-04-30 23:59:59"
static main(args) {
export(startTime, monthStartTime, endTime)
}
static void export(def startTime, def monthStartTime, def endTime) {
List<List<Integer>> mergeData = new ArrayList<>()
def sortedResult = []
sortedResult << ['项目类别', '工时', '所属项目', '项目经理']
def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"];
def result = []
List<Project> projects = ApiUtil.listProject();
for (Project project : projects) {
def ret = []
List<ProjectLabel> labels = ApiUtil.listProjectLabel(project.getId())
ret[2] = project.getName()
for (ProjectLabel label : labels) {
if ("商务负责人" == label.getGroups().getName()) {
if (ret[3] == null) {
ret[3] = label.getName()
} else {
ret[3] = ret[3] + "、" + label.getName()
}
} else if ("项目类别" == label.getGroups().getName()) {
ret[0] = label.getName()
}
}
// 获取项目下任务
int limitStart = 0
List<Task> tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50)
List<Task> tempTasks = tasks;
while (tasks != null && tempTasks != null && tempTasks.size() == 50) {
limitStart += 50
tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50)
if (CollectionUtils.isNotEmpty(tempTasks)) {
tasks.addAll(tempTasks)
}
}
if (CollectionUtils.isNotEmpty(tasks)) {
for (Task task: tasks) {
// 获取计划时间在当月的任务
TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId())
if (taskDetail == null) {
print("当前任务taskDetail为空:" + task)
Thread.sleep(60 * 1000)
taskDetail = ApiUtil.getTaskDetail(task.getId())
}
if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) {
boolean find = false
for (int i = 4; i < sortedResult[0].size(); i++) {
// 找到该项目下该负责人,则加工时
if (sortedResult[0][i] == taskDetail.getResponse().getName()) {
if (ret[i] == null) {
ret[i] = Double.valueOf(taskDetail.getFlowConten())
} else {
ret[i] += Double.valueOf(taskDetail.getFlowConten())
}
find = true
}
}
// 未找到,新增姓名
if (!find) {
sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName()
ret[sortedResult[0].size() - 1] = Double.valueOf(taskDetail.getFlowConten())
}
}
}
}
result << ret
}
def allTotal = []
for (def label : sort) {
def total = []
int startRow = sortedResult.size()
for (def ret : result) {
if (ret[0] == label) {
sortedResult << ret
for (int i = 4; i < ret.size(); i++) {
if (total[i] == null) {
total[i] = ret[i] == null ? 0 : Double.valueOf(ret[i])
} else {
total[i] += (ret[i] == null ? 0 : Double.valueOf(ret[i]))
}
}
}
}
int endRow = sortedResult.size() - 1
List<Integer> mergeRowCel = new ArrayList<>()
mergeRowCel.add(startRow)
mergeRowCel.add(endRow)
mergeRowCel.add(0)
mergeRowCel.add(0)
mergeData.add(mergeRowCel)
mergeRowCel = new ArrayList<>()
mergeRowCel.add(startRow)
mergeRowCel.add(endRow)
mergeRowCel.add(1)
mergeRowCel.add(1)
mergeData.add(mergeRowCel)
double totalNow = 0
total[2] = "小计"
sortedResult << total
for (int i = 4; i < total.size(); i++) {
if (allTotal[i] == null) {
allTotal[i] = total[i] == null ? 0 : Double.valueOf(total[i])
} else {
allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i])
}
totalNow += total[i] == null ? 0 : Double.valueOf(total[i])
}
sortedResult[startRow][1] = totalNow
}
allTotal[2] = "人员绩效总计"
sortedResult << allTotal
com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", sortedResult, "绩效统计", mergeData);
}
}
@@ -0,0 +1,134 @@
package com.fangjia.script.temp.shenjian.dingding
import com.fangjia.script.statistics.ApiUtil
import com.fangjia.script.statistics.Project
import com.fangjia.script.statistics.ProjectLabel
import com.fangjia.script.statistics.Task
import com.fangjia.script.statistics.TaskDetail
import com.fangjia.script.statistics.TaskPersonDTO
import org.apache.commons.collections.CollectionUtils
/**
* 任务统计
*
* @author shenjian
* @since 2021/5/10
*/
public class TaskExport {
static final String time = "2021年4月1日-2021年4月30日";
static final String startTime = "2021-03-01 00:00:00"
static final String monthStartTime = "2021-04-01 00:00:00"
static final String endTime = "2021-04-30 23:59:59"
static main(args) {
export(startTime, monthStartTime, endTime)
}
static void export(def startTime, def monthStartTime, def endTime) {
List<List<Integer>> mergeData = new ArrayList<>()
List<TaskPersonDTO> list = new ArrayList<>()
def result = []
result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)']
int limitStart = 0
List<Task> tasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50)
List<Task> tempTasks = tasks;
while (tasks != null && tempTasks != null && tempTasks.size() == 50) {
limitStart += 50
tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50)
if (CollectionUtils.isNotEmpty(tempTasks)) {
tasks.addAll(tempTasks)
}
}
for (Task task: tasks) {
// 获取计划时间在当月的任务
TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId())
if (taskDetail == null) {
print("当前任务taskDetail为空:" + task)
Thread.sleep(60 * 1000)
taskDetail = ApiUtil.getTaskDetail(task.getId())
}
if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) {
TaskPersonDTO dto = new TaskPersonDTO()
dto.name = task.response.name
dto.project = task.project.name
dto.taskName = task.name
dto.score = org.apache.commons.lang3.StringUtils.isBlank(taskDetail.flowConten) ? 0d : Double.valueOf(taskDetail.flowConten)
dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime))
list.add(dto)
}
}
Collections.sort(list)
Map<String, Map<String, Double>> map = new HashMap<>()
for (TaskPersonDTO dto : list) {
def ret = []
ret[0] = dto.name;
ret[1] = dto.project;
ret[2] = dto.taskName;
ret[3] = dto.score;
ret[4] = dto.workTime;
if (map.containsKey(dto.name)) {
Map<String, Double> info = map.get(dto.name)
info.put("endRow", Double.valueOf(info.get("endRow") + 1))
info.put("totalScore", Double.valueOf(info.get("totalScore") + dto.score))
info.put("workTime", Double.valueOf(info.get("workTime") + dto.workTime))
map.put(dto.name, info);
} else {
if (result.size() > 1) {
def total = []
total[2] = "小计"
total[3] = map.get(result[result.size() - 1][0]).get("totalScore")
total[4] = map.get(result[result.size() - 1][0]).get("workTime")
result << total
}
Map<String, Double> info = new HashMap<>()
info.put("startRow", Double.valueOf(result.size()))
info.put("endRow", Double.valueOf(result.size()))
info.put("startCell", 0D)
info.put("endCell", 0D)
info.put("totalScore", Double.valueOf(dto.score))
info.put("workTime", Double.valueOf(dto.workTime))
map.put(dto.name, info)
}
result << ret
}
def total = []
total[2] = "小计"
if (map.get(result[result.size() - 1][0]) != null) {
total[3] = map.get(result[result.size() - 1][0]).get("totalScore")
total[4] = map.get(result[result.size() - 1][0]).get("workTime")
} else {
total[3] = 0
total[4] = 0
print("无该信息" + result[result.size() - 1][0])
}
result << total
for (Map<String, Double> info : map.values()) {
List<Integer> data = new ArrayList<>()
data.add(info.get("startRow").toInteger())
data.add(info.get("endRow").toInteger())
data.add(info.get("startCell").toInteger())
data.add(info.get("endCell").toInteger())
mergeData.add(data)
}
com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", result, "任务统计", mergeData);
}
}
@@ -0,0 +1,25 @@
package com.fangjia.script.temp.shenjian.dto
/**
* @author shenjian
* @since 2021/5/18
*/
class AroundDTO implements Comparable<AroundDTO> {
String id;
String name;
String type
Integer distance
@Override
public int compareTo(AroundDTO o2) {
if (this.type.equals(o2.type)) {
if (this.distance != null) {
return this.distance.compareTo(o2.distance);
}
} else {
return this.type.compareTo(o2.type);
}
return 0;
}
}
@@ -0,0 +1,16 @@
package com.fangjia.script.temp.shenjian.dto
/**
* Æ¥Åä·µ»Ø½á¹¹Ìå
*/
class SearchAddressDTO {
String code;
String msg;
Result result;
class Result {
String street
String streetNumber
}
}
@@ -0,0 +1,124 @@
package com.fangjia.script.temp.shenjian.es
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
import org.apache.commons.lang3.time.DateUtils
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.*
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.reindex.DeleteByQueryRequest
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import java.util.concurrent.atomic.AtomicLong
/**
* 拷贝1.31前四周数据至3.28日缺失的四周数据中
* 注意时间更新
*
* @author shenjian
* @since 2021/3/30
*/
class CopyDataForAnnPricePre {
static RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient()
static final long timestamp = (DateUtils.parseDate("2021-03-28", "yyyy-MM-dd").getTime() -
DateUtils.parseDate("2021-01-31", "yyyy-MM-dd").getTime())
static final int days = timestamp / 1000 / 3600 / 24
static void main(String[] args) {
// 验证数据是否正确写入
//String[] weeksNew= ["2021-03-07", "2021-03-14", "2021-03-21", "2021-03-28"]
String[] weeksNew= ["2021-03-07", "2021-01-31"]
for (String week : weeksNew) {
println(DateUtils.parseDate(week, "yyyy-MM-dd").getTime())
}
//String[] weeksNew=["2021-03-28"]
// ///////////////////// 删除数据临时逻辑
// // 先删除数据
// DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("ann_price_pre")
// deleteRequest.setQuery(QueryBuilders.termQuery("date", DateUtils.parseDate("2021-03-07", "yyyy-MM-dd").getTime()))
// // 批次大小
// deleteRequest.setBatchSize(1000);
// restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT)
// /////////////////////
checkNum(weeksNew)
BulkRequest request = new BulkRequest();
for (String week : weeksNew) {
AtomicLong count = new AtomicLong(0)
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.addDays(DateUtils.parseDate(week, "yyyy-MM-dd"), -days).getTime()))
.size(10000);
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
SearchRequest searchRequest = new SearchRequest("ann_price_pre")
.source(searchSourceBuilder)
.scroll(scroll); // 设置scroll间隔
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); //通过发送初始搜索请求来初始化搜索上下文
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits()
write(count, searchHits, timestamp, request);
while (searchHits != null && searchHits.length > 0) { //通过循环调用搜索滚动api来检索所有搜索命中,直到没有文档返回
//处理返回的搜索结果
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); //创建一个新的搜索滚动请求,保存最后返回的滚动标识符和滚动间隔
scrollRequest.scroll(scroll);
searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
write(count, searchHits, timestamp, request);
}
if (count.get() > 0) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT)
println(week + "剩余批量操作成功:" + bulkResponse.status())
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); //完成滚动后,清除滚动上下文
clearScrollRequest.addScrollId(scrollId);
ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
boolean succeeded = clearScrollResponse.isSucceeded();
println("清除滚动句柄" + succeeded)
}
checkNum(weeksNew)
restHighLevelClient.close()
}
static write(def count, def searchHits, def timestamp, def request) {
for (SearchHit hit : searchHits) {
Map<String, Object> map = hit.getSourceAsMap();
map.put("date", Long.valueOf(map.get("date").toString()) + timestamp)
request.add(new IndexRequest("ann_price_pre").source(map).opType(DocWriteRequest.OpType.CREATE));
count.incrementAndGet()
if (count.get() >= 5000) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
println("批量操作成功:" + bulkResponse.status())
request = new BulkRequest();
count = new AtomicLong(0)
}
}
}
static checkNum(def weeksNew) {
// 验证数据是否正确写入
for (String week : weeksNew) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.parseDate(week, "yyyy-MM-dd").getTime()));
SearchRequest searchRequest = new SearchRequest("ann_price_pre")
searchRequest.source(searchSourceBuilder)
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
println(searchResponse.status().name() + " " + searchResponse.getHits().totalHits)
}
}
}
@@ -0,0 +1,98 @@
package com.fangjia.script.temp.shenjian.es
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 由于快照恢复字段类型错误
* 同步es6.4该索引至es7.11,先写入mongo
*
* @author shenjian
* @since 2021/3/30
*/
class SyncDistrictPriceStat {
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017)
static void main(String[] args) {
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery()
def districtPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder)
//def newDistrictPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder)
// RestHighLevelClient restHighLevelClient206 = getClient206()
// RestHighLevelClient restHighLevelClient = getClient()
// //def districtPriceStats
// GetResponse getResponse = getDocument(restHighLevelClient)
for (def map : districtPriceStats) {
DBObject dbObject = new BasicDBObject()
dbObject.put("badcase_count", map.badcase_count)
dbObject.put("city", map.city)
dbObject.put("city_grade", map.city_grade)
dbObject.put("credit_district_count", map.credit_district_count)
dbObject.put("date", map.date)
dbObject.put("district_count", map.district_count)
dbObject.put("price_district_count", map.price_district_count)
dbObject.put("score", map.score)
dbObject.put("vote_district", map.vote_district)
fangjiaConn.save(dbObject)
// // 创建索引请求
// IndexRequest request = new IndexRequest("district_price_stat")
// request.id(id).timeout(TimeValue.timeValueSeconds(1))
// request.source(map)
//
// // 将请求发出去,获取响应结果
// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT);
//
// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE
// System.out.println(indexResponse.toString()); //响应内容
}
//restHighLevelClient.close()
}
// static RestHighLevelClient getClient() {
// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.216",9200,"http"))
// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
// @Override
// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
// httpClientBuilder.disableAuthCaching();
// CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
// return httpClientBuilder;
// }
// }))
// return restHighLevelClient
// }
//
// static RestHighLevelClient getClient206() {
// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.206",9200,"http"))
// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
// @Override
// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
// httpClientBuilder.disableAuthCaching();
// CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
// return httpClientBuilder;
// }
// }))
// return restHighLevelClient
// }
//
// static GetResponse getDocument(RestHighLevelClient restHighLevelClient206) throws IOException {
// SearchRequest searchRequest = new SearchRequest("district_price_stat");
// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// //searchSourceBuilder.query(QueryBuilders.termQuery("url", "test"));
// searchRequest.source(searchSourceBuilder);
// SearchResponse response = restHighLevelClient206.search(searchRequest, RequestOptions.DEFAULT);
// System.out.println(response.getSourceAsString()); //获取文档的内容
// return response
// }
}
@@ -0,0 +1,78 @@
package com.fangjia.script.temp.shenjian.es
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
import com.mongodb.DBCollection
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.reindex.DeleteByQueryRequest
import java.util.concurrent.atomic.AtomicLong
/**
* 由于快照恢复字段类型错误
* 同步es6.4该索引至es7.11
*
* @author shenjian
* @since 2021/3/30
*/
class SyncDistrictPriceStat2 {
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017)
static void main(String[] args) {
RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient()
// 先删除数据
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("district_price_stat")
deleteRequest.setQuery(QueryBuilders.matchAllQuery())
// 批次大小
deleteRequest.setBatchSize(1000);
restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT)
BulkRequest request = new BulkRequest();
AtomicLong count = new AtomicLong()
fangjiaConn.find().batchSize(10000).each {map ->
Map dbObject = new HashMap()
dbObject.put("badcase_count", map.badcase_count)
dbObject.put("city", map.city)
dbObject.put("city_grade", map.city_grade)
dbObject.put("credit_district_count", map.credit_district_count)
dbObject.put("date", map.date)
dbObject.put("district_count", map.district_count)
dbObject.put("price_district_count", map.price_district_count)
dbObject.put("score", map.score)
dbObject.put("vote_district", map.vote_district)
request.add(new IndexRequest("district_price_stat").source(dbObject).opType(DocWriteRequest.OpType.CREATE));
count.incrementAndGet()
// 每一千写入
if (count.get() >= 1000) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
request = new BulkRequest();
count = new AtomicLong(0)
}
// 单个插入文档
// // 创建索引请求
// IndexRequest request = new IndexRequest("district_price_stat")
// request.timeout(TimeValue.timeValueSeconds(1))
// request.source(dbObject)
//
// // 将请求发出去,获取响应结果
// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT);
// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE
// System.out.println(indexResponse.toString()); //响应内容
}
// 不足一千写入
if (count.get() > 0) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
}
restHighLevelClient.close()
}
}
@@ -0,0 +1,35 @@
package com.fangjia.script.temp.shenjian.es.util
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
/**
* @author shenjian
* @since 2021/3/30
*/
class RestHighLevelClientUtils {
static final def HOST = "192.168.0.216"
static RestHighLevelClient getClient() {
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(HOST,9200,"http"))
.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpClientBuilder;
}
}))
return restHighLevelClient
}
}
@@ -0,0 +1,116 @@
package com.fangjia.script.temp.shenjian.fill
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.temp.shenjian.util.DistrictUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.GeoHelper
import groovy.util.logging.Slf4j
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 根据周围500米容积率与绿化率平均值推算小区
*
* @author shenjian*
* @since 2021-03-13
*/
@Slf4j
class CalcCubageAndVirescence {
static main(args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("CalcCubageAndVirescence")
log.info("开始根据周围500米容积率与绿化率平均值推算小区")
def cities = ['北京', '重庆', '哈尔滨']
for (def city : cities) {
def sameRegionDistricts = [:]
def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts);
def result = []
districts.each { obj ->
if (obj.finish_building_date) {
int year = CommonUtils.getBuildYear(obj.finish_building_date)
if (year > 2000) {
if (!(obj.cubage_percent && obj.virescence_percent)) {
def temp = this.findResult(obj.lat2 / 10000000000.0d, obj.lng2 / 10000000000.0d, 500, sameRegionDistricts.get(obj.region), obj.name, obj._id)
result << temp
}
}
}
}
def headers = ["小区ID", "容积率", "绿化率"]
FileUtils.batchCreateFile(result, "d:\\数据\\小区\\补充容积绿化" + city + ".csv", headers, "UTF-8")
}
stopWatch.stop()
log.info("完成根据周围500米容积率与绿化率平均值推算, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
}
static def findResult(lat, lng, radius, districts, region, id) {
def avgMap = [:]
def ret = [id]
if (lat && lng) {
def cubagePercentArr = []
def virescencePercentArr = []
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, radius)
for (def row : districts) {
if (row.lat2 && row.lng2) {
def lat2 = row.lat2 / 10000000000.0;
def lng2 = row.lng2 / 10000000000.0;
if (lat2 >= map.minLat && lat2 <= map.maxLat && lng2 >= map.minLng && lng2 <= map.maxLng) {
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
if (distance <= radius) {
if (row.cubage_percent) {
cubagePercentArr << row.cubage_percent;
}
if (row.virescence_percent) {
virescencePercentArr << row.virescence_percent
}
}
}
}
}
try {
// 计算平均容积率
if (cubagePercentArr) {
double sum = 0
double count = 0.0d
for (def m : cubagePercentArr) {
sum = sum + Double.valueOf(m)
count++
}
if (sum) {
double avgCubagePercent = sum / count
ret << DataHandleUtils.getDoubleRound(2, avgCubagePercent)
}
} else {
ret << ""
}
// 计算平均绿化率
if (virescencePercentArr) {
double sum = 0
double count = 0.0d
for (def m : virescencePercentArr) {
sum += Double.valueOf(m)
count++
}
if (sum) {
def avgVirescencePercent = sum / (count)
ret << DataHandleUtils.getDoubleRound(2, avgVirescencePercent)
}
} else {
ret << ""
}
log.info("找到可以优化的小区" + region + "周围平均值为:容积率" + ret[1] + ", 绿化率" + ret[2])
} catch(Exception ex) {
log.error("容积率或绿化率不标准,无法解析")
}
}
return ret
}
}
@@ -0,0 +1,46 @@
package com.fangjia.script.temp.shenjian.poi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 商圈经纬度
*/
class BusinessLatLng {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018)
static ExecutorService executorService = Executors.newFixedThreadPool(1)
static void main(String[] args) {
dealPoi(new BasicDBObject("city", "上海").append("region", "浦东新区"))
executorService.shutdown()
}
static dealPoi(DBObject query) {
collection.find(query).batchSize(5000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def city = obj.city
def address = city + obj.region + obj.address;
def location = GaoDeUtils.parserByGaode(city, address);
if (location && location.lng) {
def lng = Double.valueOf(location.lng)
def lat = Double.valueOf(location.lat)
collection.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0)
.append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "已获取经纬度")))
} else {
collection.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空")))
}
println("已处理小区:" + obj)
}
})
}
}
}
@@ -0,0 +1,44 @@
package com.fangjia.script.temp.shenjian.poi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 国家标准居委会经纬度geohash化
*
* @author shenjian
* @since 2021/7/22
*/
class CommitteeGeoHash {
static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
dealPoi(new BasicDBObject("status_location", 0).append("lng", new BasicDBObject('$ne', null)))
executorService.shutdown()
}
static dealPoi(DBObject query) {
villagesConn.find(query).batchSize(5000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
double lng = Double.valueOf(obj.lng)
double lat = Double.valueOf(obj.lat)
def geohash = SeaweedUtils.encodeByGeoHash(lng, lat, 8);
villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("geohash", geohash)))
println("已处理小区:" + obj)
}
})
}
}
}
@@ -0,0 +1,119 @@
package com.fangjia.script.temp.shenjian.poi
import com.fangjia.core.util.CosineSimilarAlgorithm
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 国家标准居委会经纬度获取
*
* @author shenjian
* @since 2021/7/22
*/
class CommitteeLatLng {
static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017)
static ExecutorService executorService = Executors.newFixedThreadPool(20)
static void main(String[] args) {
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("name", "席庄二村委会"))
//dealLocation()
executorService.shutdown()
}
static dealPoi(DBObject query) {
villagesConn.find(query).batchSize(5000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def province_name = obj.province_name
def city = obj.city_name
def area = obj.area_name
def name = obj.name
if (city == "市辖区") {
city = obj.province_name
} else if (city == "省直辖县级行政区划" || city == "自治区直辖县级行政区划") {
city = obj.area_name
area = ""
} else if (city == "县") {
city = obj.province_name
province_name = ""
}
if (obj.name.contains("虚拟生活区") || obj.name.contains("管理区") || obj.name.contains("虚拟社区")) {
name = obj.street_name
}
if (name.contains("社区居民委员会")) {
name = name.replace("居民委员会", "")
}
if (name.contains("村村委会")) {
name = name.replace("村委会", "")
} else if (name.contains("村村民委员会")) {
name = name.replace("村民委员会", "")
} else if (name.contains("村委会")) {
name = name.replace("村委会", "村")
} else if (name.contains("村民委员会")) {
name = name.replace("村民委员会", "村")
}
String address = province_name + " " + city + " " + area + " " + obj.street_name + " " + name;
//String address = province_name + city + area + name
List<AmapPoi> amapPoi = GaoDeUtils.searchPoi(city, address, area, "130106|130105|130104")
if (amapPoi && amapPoi.size() > 0) {
def poi = amapPoi[0]
double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, name)
if (similarity <= 0.8) {
amapPoi = GaoDeUtils.searchPoi(city, address, area, "")
}
poi = amapPoi[0]
String[] locations = poi.location.split(",")
def lng = Double.valueOf(locations[0])
def lat = Double.valueOf(locations[1])
villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0)
.append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度")))
} else {
amapPoi = GaoDeUtils.searchPoi(city, address, area, "")
if (amapPoi && amapPoi.size() > 0) {
def poi = amapPoi[0]
String[] locations = poi.location.split(",")
def lng = Double.valueOf(locations[0])
def lat = Double.valueOf(locations[1])
villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0)
.append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度")))
} else {
villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空")))
}
}
println("已处理小区:" + obj)
}
})
}
}
static dealLocation() {
villagesConn.find(new BasicDBObject("status_location", 0)).batchSize(5000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
if (obj.lng && obj.lat) {
def lng = Double.valueOf(obj.lng)
def lat = Double.valueOf(obj.lat)
villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("location", new BasicDBObject("lat", lat).append("lon", lng))))
}
println("已处理小区:" + obj)
}
})
}
}
}
@@ -0,0 +1,30 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.coredata.util.CollectionUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
/**
* 反查高德小区信息
*/
class CounterCheckGaoDe {
static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
List<List<String>> infos = ExcelUtils.read("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\地图反查信息_高德结果2.xlsx")
for (List<String> info : infos) {
if (info.size() < 6) {
String id = info.get(4)
List<AmapPoi> amapPois = GaoDeUtils.getAmapById(id)
if (CollectionUtils.isNotEmpty(amapPois)) {
String jsonStr = JSONObject.toJSONString(amapPois)
HttpUtils.postForJson(URL, jsonStr)
}
}
}
}
}
@@ -0,0 +1,71 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 获取具体点poi
*
* @author shenjian
* @since 2021/7/26
*/
class GetPoi {
static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi"
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
List<List<String>> seaweeds = ExcelUtils.read("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\分期数据高德核实.xlsx")
for (List<String> infoTemp : seaweeds) {
final List<String> info = infoTemp
executorService.submit(new Runnable() {
@Override
void run() {
String city = info.get(2)
String region = info.get(3)
String seaweedName = info.get(4)
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302");
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
atomicLong.incrementAndGet()
}
synchronized (this) {
if (atomicLong.intValue() % 500 == 0) {
println(atomicLong.intValue())
}
}
}
})
}
executorService.shutdown()
}
static void getPointInfo() {
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi("无锡","190301",120.411465,31.864912, 1000, "江苏省江阴市周庄镇兴隆北路");
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
}
}
}
@@ -0,0 +1,74 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.excel.EasyExcel
import com.fangjia.script.coredata.util.CollectionUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 学校经纬度获取
*
* @author shenjian
* @since 2021/8/16
*/
class SchoolLatLng {
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static String name = "大连市义务教育学校名录test";
static void main(String[] args) {
List<List<String>> schools = ExcelUtils.read(new FileInputStream("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + ".xlsx"));
Map<String, List<String>> result = new ConcurrentHashMap<>();
CountDownLatch latch = new CountDownLatch(schools.size());
for (List<String> schoolTemp : schools) {
final List<String> school = schoolTemp
executorService.submit(new Runnable() {
@Override
void run() {
try {
def city = "大连"
def list = new ArrayList();
def address = school.get(1)
def pois = GaoDeUtils.searchPoi(city, address);
if (CollectionUtils.isNotEmpty(pois)) {
for (AmapPoi amapPoi : pois) {
if (address.contains(amapPoi.getName())) {
String[] locations = amapPoi.getLocation().split(",")
list.add(locations[0])
list.add(locations[1])
list.add(amapPoi.getId())
result.put(address, list)
break;
}
}
}
println("已处理:" + address)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
List<List<String>> head = new ArrayList<>()
for (List<String> school : schools) {
def address = school.get(1)
if (result.contains(address)) {
school.addAll(result.get(address))
}
}
EasyExcel.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + "结果.xlsx").head(head).sheet("学校").doWrite(schools)
executorService.shutdown()
}
}
@@ -0,0 +1,58 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区周边居委会获取
*
* @author shenjian
* @since 2021/5/24
*/
class SeaweedAroundCommittee {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
// 乡镇以下级政府及事业单位(居委会)
def types = "130106"
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
}
static dealPoi(DBObject query, def types) {
query.put("status_committee", 1)
conn136.find(query).batchSize(5000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_committee", 0).append("flag_committee", "已获取居委会")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_committee", 2).append("flag_committee", "周边居委会为空")))
}
println("已处理小区:" + obj)
}
})
}
}
}
@@ -0,0 +1,191 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区周边POI数据获取
*
* @author shenjian
* @since 2021/5/24
*/
@BatchOwner(Watcher.ShenJian)
class SeaweedAroundPoi extends BaseBatch {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
def types = listAllType()
//dealSpecialSeaweedAround(types)
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
}
/**
* 有限处理招标城市
*/
static dealSpecialSeaweedAround(def types) {
def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"]
for (def city : cities) {
DBObject q = new BasicDBObject();
q.put("city", city);
if (city == "福安") {
city = "宁德"
q.put("city", city)
q.put("region", "福安")
} else if (city == "平潭") {
city = "福州"
q.put("city", city)
q.put("region", "平潭")
} else if (city == "石狮") {
city = "泉州"
q.put("city", city)
q.put("region", "石狮")
}
dealPoi(q, types)
}
}
static dealPoi(DBObject query, def types) {
query.put("status", 1)
conn136.find(query).batchSize(5000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
})
}
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List list = []
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
// 已存在小区,无需同步
} else {
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
list << newObj
if (list.size() == 5000) {
conn136.insert(list)
list.clear()
}
}
}
conn136.insert(list)
list.clear()
}
static def getByIdAndInsert(String id) {
BasicDBObject obj = DbUtils.seaweed.findById(id);
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
conn136.insert(newObj)
}
public static def listAllType() {
StringBuilder ret = new StringBuilder()
for (def val : KEYWORDS_MAP.values()) {
ret.append(val).append("|")
}
String result = ret.toString()
return result.substring(0, result.length() - 1)
}
public static final Map<String, String> KEYWORDS_MAP = new HashMap<String, String>() {
{
put("餐饮", "05");
// put("地铁", "150500");
// put("公交", "1507");
// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
// put("写字楼", "120201");
// put("学校", "141200|141201|141202|141203|141204");
// put("银行", "1601|1603");
// put("大型超市", "0604");
// put("商场", "0601");
// put("餐饮", "0501|0502|0503|0504");
// put("旅游景点", "110000");
// put("图书馆", "140500");
// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107");
// put("丧葬设施", "071900|071901|071902|071903")
}
};
@Override
void process(Object args) {
def types = listAllType()
//dealSpecialSeaweedAround(types)
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
}
}
@@ -0,0 +1,128 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区周边POI数据获取单线程方式
*
* @author shenjian
* @since 2021/5/24
*/
class SeaweedAroundPoiSingal {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
def types = listAllType()
//def types = "0601"
DBObject query = new BasicDBObject()
query.put("status", 1)
conn136.find(query).batchSize(30).each {objTemp ->
final def obj = objTemp
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List list = []
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
// 已存在小区,无需同步
} else {
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
list << newObj
if (list.size() == 5000) {
conn136.insert(list)
list.clear()
}
}
}
conn136.insert(list)
list.clear()
}
static def getByIdAndInsert(String id) {
BasicDBObject obj = DbUtils.seaweed.findById(id);
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
conn136.insert(newObj)
}
static def listAllType() {
StringBuilder ret = new StringBuilder()
for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) {
ret.append(val).append("|")
}
String result = ret.toString()
return result.substring(0, result.length() - 1)
}
}
@@ -0,0 +1,181 @@
package com.fangjia.script.temp.shenjian.poi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 对口小学初中经纬度
*/
class SeaweedSchoolLatLng {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018)
static DBCollection school_collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018)
static ExecutorService executorService = Executors.newFixedThreadPool(20)
static void main(String[] args) {
//latLng()
updateAttribute()
executorService.shutdown()
}
static void updateAttribute() {
AtomicLong atomicLong = new AtomicLong(0);
collection.find(new BasicDBObject()).batchSize(1000).each {objT ->
final DBObject obj = objT
executorService.submit(new Runnable() {
@Override
void run() {
try {
def address = obj.address
boolean update = false
if (address.contains("浦东")) {
BasicDBList primarySchools = obj.primarySchool
BasicDBList middleSchools = obj.middleSchool
DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'district_name': obj.name))
def quality = ''
if (primarySchools != null) {
if (schoolObj && schoolObj.attribute) {
quality = schoolObj.attribute
}
for (BasicDBObject dbObject : primarySchools) {
if (dbObject.name == '第六师范学校第二附属小学西校区' || String.valueOf(dbObject.name).startsWith('第六师范附属小学')
|| dbObject.name == '上海市第六师范第二附属小学七号桥校区' || dbObject.name == '康城学校康佳校区' || dbObject.name == '三墩学校') {
dbObject.quality = '普通'
update = true
} else if (dbObject.name == '上海市实验学校东校(小学)' || dbObject.name == '浦东新区竹园小学沪东校区') {
dbObject.quality = '市重点'
update = true
} else if (dbObject.name == '浦东新区上南二村小学南校区' || dbObject.name == '浦东新区新时代小学大华校区'
|| dbObject.name == '浦东新区明珠小学A区') {
dbObject.quality = '区重点'
update = true
}
else if (quality != '' && schoolObj.category == '小学') {
dbObject.quality = quality
update = true
} else {
update = updateDbObjectQuality(dbObject);
}
}
}
def update2 = false
if (middleSchools != null) {
for (BasicDBObject dbObject : middleSchools) {
if (dbObject.name == '华东师范大学张江实验中学' || dbObject.name == '10上海市实验学校(初中)(全市招生)' || dbObject.name == '上海市实验学校附属光明学校总校区'
|| dbObject.name == '上海市浦东新区进才实验中学南校' || String.valueOf(dbObject.name).startsWith('建平中学') || String.valueOf(dbObject.name).startsWith('建平实验中学')
|| dbObject.name == '张江集团中学' || dbObject.name == '洋泾-菊园实验学校' || dbObject.name == '浦东外国语学校东校' || dbObject.name == '浦东新区建平中学南校' || String.valueOf(dbObject.name).startsWith('育人中学')
|| dbObject.name == '致远中学' || String.valueOf(dbObject.name).startsWith('进才中学') || String.valueOf(dbObject.name).startsWith('进才实验中学')) {
dbObject.quality = '市重点'
update2 = true
} else if (dbObject.name == '华东师范大学附属东昌中学南校潍坊校区' || dbObject.name == '下沙学校鹤沙路校区' || dbObject.name == '傅雷中学关岳路校区'
|| dbObject.name == '傅雷中学瑞和路校区' || dbObject.name == '华东师范大学附属东昌中学南校张江校区' || dbObject.name == '南汇第一中学卫星东路校区'
|| dbObject.name == '康城学校(拯安校区)(初中)' || dbObject.name == '建平香梅中学' || dbObject.name == '沪新中学' || dbObject.name == '浦东模范中学'
|| dbObject.name == '浦东模范中学东校' || dbObject.name == '浦东模范实验中学' || String.valueOf(dbObject.name).startsWith('澧溪中学') ||
dbObject.name == '清流中学' || String.valueOf(dbObject.name).startsWith('王港中学') || dbObject.name == '罗山中学'
|| dbObject.name == '金杨中学' || dbObject.name == '香山中学' || dbObject.name == '高行中学') {
dbObject.quality = '区重点'
update2 = true
} else if (quality != '' && schoolObj.category == '初中') {
dbObject.quality = quality
update2 = true
} else {
update2 = updateDbObjectQuality(dbObject)
}
}
}
if (update || update2) {
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true)
}
}
} catch(Exception ex) {
ex.printStackTrace()
} finally {
atomicLong.incrementAndGet()
synchronized (this) {
if (atomicLong.longValue() % 100 == 0) {
println(atomicLong.longValue())
}
}
}
}
})
};
}
static boolean updateDbObjectQuality(BasicDBObject dbObject) {
if (dbObject.quality == '公办') {
DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'school_name': dbObject.name))
if (schoolObj && schoolObj.attribute) {
dbObject.quality = schoolObj.attribute
return true
}
}
return false
}
static void latLng() {
AtomicLong atomicLong = new AtomicLong(0);
collection.find(new BasicDBObject()).batchSize(1000).each {objT ->
final DBObject obj = objT
executorService.submit(new Runnable() {
@Override
void run() {
try {
def address = obj.address
if (address.contains("浦东")) {
BasicDBList primarySchools = obj.primarySchool
BasicDBList middleSchools = obj.middleSchool
if (primarySchools != null) {
for (BasicDBObject dbObject : primarySchools) {
def schoolAddress = dbObject.address
if (dbObject.lng == null) {
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress)
if (location && location.lng) {
def lng = Double.valueOf(location.lng)
def lat = Double.valueOf(location.lat)
dbObject.lng = lng
dbObject.lat = lat
}
}
}
}
if (middleSchools != null) {
for (BasicDBObject dbObject : middleSchools) {
def schoolAddress = dbObject.address
if (dbObject.lng == null) {
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress)
if (location && location.lng) {
def lng = Double.valueOf(location.lng)
def lat = Double.valueOf(location.lat)
dbObject.lng = lng
dbObject.lat = lat
}
}
}
}
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true)
}
} catch(Exception ex) {
ex.printStackTrace()
} finally {
atomicLong.incrementAndGet()
synchronized (this) {
if (atomicLong.longValue() % 100 == 0) {
println(atomicLong.longValue())
}
}
}
}
})
};
}
}
@@ -0,0 +1,59 @@
package com.fangjia.script.temp.shenjian.poi
import com.fangjia.script.temp.shenjian.util.MyStringUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
/**
* @author shenjian* @since 2021/7/7
*/
class StatisticsPoi {
static final cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京",
"青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原",
"台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳",
"哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江",
"岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳",
"柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照",
"拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
static final types = ['地铁', '公交', '医院', '写字楼', '学校', '银行', '大型超市', '商场', '餐饮', '旅游景点', '图书馆']
static void main(String[] args) {
def result = []
result << ['城市', '数据量', '地铁', '公交', '医院', '写字楼', '学校', '银行', '大型超市', '商场', '餐饮', '旅游景点', '图书馆']
for (def city in cities) {
def ret = []
def total
String sql = "select count(*) total from amap_poi where cityname = '" + city + "' and status <> 1 "
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.total
})
ret << city
ret << total
for (def type in types) {
def typeCode = SeaweedAroundPoi.KEYWORDS_MAP.get(type)
def typeCodeArr = typeCode.split("\\|")
String val = ""
String sqlType = ""
if (typeCodeArr[0].size() <= 4) {
val = MyStringUtils.joinLikeSql(typeCodeArr)
sqlType = "select count(*) total from amap_poi where cityname = '" + city + "' and (" + val + ") and status <> 1 "
} else {
val = MyStringUtils.joinInSql(typeCodeArr)
sqlType = "select count(*) total from amap_poi where cityname = '" + city + "' and typecode in (" + val + ") and status <> 1 "
}
ElasticUtils.executeSqlQuery(sqlType,{ data->
total = data.total
})
ret << total
}
result << ret
println(city)
}
ExcelUtils.write("F:\\工作\\房产估值模型及数据招标\\交付工作\\POI信息抓取情况总结.xlsx", result, "POI统计")
}
}
@@ -0,0 +1,69 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.excel.EasyExcel
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import joptsimple.internal.Strings
import org.apache.commons.collections.CollectionUtils
/**
* 银联小区经纬度与周边配套
*/
class YinlianLngLatAndAround {
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
this.getAround()
}
public static void getAround() {
List<List<String>> datas = ExcelUtils.read("D:\\数据\\银联\\测试样本.xlsx")
for (List<String> data : datas) {
if ("城市".equals(data.get(0))) {
continue;
}
if (data.size() == 7) {
String city = data.get(0)
double lng = Double.valueOf(data.get(5))
double lat = Double.valueOf(data.get(6))
HttpUtils.postForJson()
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, SeaweedAroundPoi.listAllType(), lng, lat, 3000, "")
if (CollectionUtils.isNotEmpty(pois)) {
// String jsonStr = JSONObject.toJSONString(pois)
// HttpUtils.postForJson(URL, jsonStr)
List<String> infos = new ArrayList<>()
for (AmapPoi amapPoi : pois) {
infos.add(amapPoi.getName())
}
data.add(Strings.join(infos, ","))
}
}
}
def headArr = ["城市","区域","小区名","面积","估值","周边3公里配套", "经度", "纬度"]
def head = ExcelUtils.formatHead(headArr)
EasyExcel.write("D:\\数据\\银联\\测试样本-估值&小区配套3公里(银联测试)结果.xlsx").head(head).sheet("Sheet1").doWrite(datas);
}
public void getLocation() {
List<List<String>> datas = ExcelUtils.read("D:\\数据\\银联\\测试样本.xlsx")
for (List<String> data : datas) {
String city = data.get(0)
String region = data.get(1)
String name = data.get(2)
String address = city + region + name
List<Double> lngLat = GaoDeUtils.getLngLat(city, address, region, name, "")
if (CollectionUtils.isNotEmpty(lngLat)) {
data.add(lngLat.get(0))
data.add(lngLat.get(1))
}
}
def headArr = ["城市","区域","小区名","面积","估值","周边3公里配套", "经度", "纬度"]
def head = ExcelUtils.formatHead(headArr)
EasyExcel.write("D:\\数据\\银联\\测试样本-估值&小区配套3公里(银联测试)结果.xlsx").head(head).sheet("Sheet1").doWrite(datas);
}
}
@@ -0,0 +1,48 @@
[
{
"id": "B0FFLCY016",
"parent": [],
"childtype": [],
"name": "杭州馨悦坊公寓",
"type": "住宿服务;住宿服务相关;住宿服务相关",
"typecode": "100000",
"biz_type": "hotel",
"address": "18号大街伊萨卡浩泽园",
"location": "120.383947,30.289931",
"tel": [],
"pname": "浙江省",
"cityname": "杭州市",
"adname": "钱塘区",
"importance": [],
"shopid": [],
"shopinfo": "0",
"poiweight": [],
"distance": "992",
"biz_ext": {
"rating": [],
"cost": [],
"hotel_ordering": "0",
"lowest_price": [],
"star": [],
"open_time": [],
"opentime2": []
},
"photos": [
{
"title": "Logo",
"url": "http://store.is.autonavi.com/showpic/7e9c5bdeeba929eb4742102b7c9bd47c",
"provider": []
},
{
"title": "客房",
"url": "http://store.is.autonavi.com/showpic/e4e5771e4de5083877931fda2e360f74",
"provider": []
},
{
"title": [],
"url": "http://store.is.autonavi.com/showpic/5401e1e105bcbacf550896c7575ee315",
"provider": []
}
]
}
]
@@ -0,0 +1,79 @@
package com.fangjia.script.temp.shenjian.school
import com.fangjia.core.util.AddressLatlng
import com.fangjia.script.district.Distance
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.bson.types.ObjectId
/**
* 处理小区-学校距离过大的数据
*
* @author shenjian
* @since 2021/4/16
*/
class DistanceDeal {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018)
// 修复 鑫苑国际城市花园 https://guojichengshihuayuanszxy.fang.com 区域问题
static void main(String[] args) {
def result = GaoDeApiUtils.geocode("117.363998413086,31.89208984375")
print(result)
/* def ret = AddressLatlng.getLngLatByAddress("苏州", "苏州工业园区星湾学校(西校区)")
print(ret)*/
// DBObject updateField = new BasicDBObject()
// updateField.put("school_lng", "120.68764")
// updateField.put("school_lat", "31.330963")
// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10483.htm"), new BasicDBObject('$set', updateField), false, true)
double length = Distance.GetDistance();
print(length)
// DBObject updateField = new BasicDBObject()
// updateField.put("school_lng", 120.68764)
// updateField.put("school_lat", 31.330963)
// updateField.put("school_name", "星湾学校西校区")
// updateField.put("school_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上")
// updateField.put("gaode_name", "星湾学校西校区")
// updateField.put("gaode_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上")
//
// collection.update(new BasicDBObject("match_name", new BasicDBObject('$in', ["星海人家", "新加花园", "中央景城", "加城湖滨公寓", "沁苑小区", "湖畔花园",
// "中茵皇冠国际", "天域三期", "天域", "天域二期"])).append("match_region", "吴中"), new BasicDBObject('$set', updateField), false, true)
// def location = this.parserByGaode("苏州", "苏州工业园区星湾学校(西校区)")
// print(location)
// if (location != null) {
//DBObject updateField = new BasicDBObject()
// updateField.put("match_lng", "120.614203")
// updateField.put("match_lat", "31.309876")
// collection.update(new BasicDBObject("url", "https://jingdelu0512.fang.com"), new BasicDBObject('$set', updateField), false, true)
//
// updateField.put("match_lng", "120.610651479")
// updateField.put("match_lat", "31.3131018333")
// collection.update(new BasicDBObject("url", "https://wuqufang.fang.com"), new BasicDBObject('$set', updateField), false, true)
//
// updateField.put("match_lng", "117.2703018188")
// updateField.put("match_lat", "31.7484989166")
// collection.update(new BasicDBObject("url", "https://binghuqiancheng.fang.com"), new BasicDBObject('$set', updateField), false, true)
//
// updateField.put("match_lng", "120.610343")
// updateField.put("match_lat", "31.312715")
// collection.update(new BasicDBObject("url", "https://baolinsiqian.fang.com"), new BasicDBObject('$set', updateField), false, true)
// }
// DBObject updateField = new BasicDBObject()
// updateField.put("match_region", "蜀山")
// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10689.htm"), new BasicDBObject('$set', updateField), false, true)
// }
}
}
@@ -0,0 +1,101 @@
package com.fangjia.script.temp.shenjian.school
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang3.StringUtils
/**
* 填充规则
*
* 1.中学信息按期次填充
*
* 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。
* 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样
*
* @author shenjian
* @since 2021/4/16
*/
class FillMidSchoolInfo {
static void main(String[] args) {
Map<String, List<Object>> map = new HashMap<>();
Map<String, List<Object>> map2 = new HashMap<>();
List<List<Object>> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx")
for (List<Object> data : list) {
String city = data.get(0)
String region = data.get(1)
String districtName = String.valueOf(data.get(2))
String key = city + region
if (districtName.contains("期")) {
key += districtName.substring(0, districtName.length() - 2);
if (StringUtils.isNotBlank(data.get(11))) { // 存在对口初中
map.put(key, data);
}
if (StringUtils.isNotBlank(data.get(6))) { // 存在对口小学
map2.put(key, data);
}
} else {
key += districtName
if (StringUtils.isNotBlank(data.get(11))) { // 存在对口初中
map.put(key, data);
}
if (StringUtils.isNotBlank(data.get(6))) { // 存在对口小学
map2.put(key, data);
}
}
}
print(map.size())
Long countPrimary = 0;
Long count = 0;
for (List<Object> data : list) {
String city = data.get(0)
String region = data.get(1)
String districtName = String.valueOf(data.get(2))
String key = city + region
if (districtName.contains("期")) {
key += districtName.substring(0, districtName.length() - 2)
} else {
key += districtName
}
if (map.get(key) != null) {
List<Object> val = map.get(key)
if (StringUtils.isBlank(data.get(11))) {
data.set(11, val.get(11))
data.set(12, val.get(12))
data.set(13, val.get(13))
data.set(14, val.get(14))
data.set(15, val.get(15))
count++
println("初中" + data)
}
}
if (map2.get(key) != null) {
List<Object> val = map2.get(key)
if (StringUtils.isBlank(data.get(6))) {
data.set(6, val.get(6))
data.set(7, val.get(7))
data.set(8, val.get(8))
data.set(9, val.get(9))
data.set(10, val.get(10))
countPrimary++
println("小学" + data)
}
}
}
List<List<Object>> result = new ArrayList<>();
for (List<Object> data : list) {
if (StringUtils.isNotBlank(data.get(6))) {
result.add(data)
}
}
print("优化数量" + countPrimary + " " + count)
String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站",
"小区ID","数据类型"];
List<String> headerList = Arrays.asList(header)
result.add(0, headerList)
ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", result, "数据")
}
}
@@ -0,0 +1,57 @@
package com.fangjia.script.temp.shenjian.school
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang3.StringUtils
/**
* 填充规则
*
* 1.中学信息按期次填充
*
* 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。
* 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样
*
* @author shenjian
* @since 2021/4/16
*/
class FillMidSchoolWithPrimarySchool {
static void main(String[] args) {
Map<String, List<Object>> map = new HashMap<>();
List<List<Object>> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx")
for (List<Object> data : list) {
// 对口小学 对口中学
String city = data.get(0)
String region = data.get(1)
if (org.apache.commons.lang.StringUtils.isNotBlank(data.get(7)) && StringUtils.isNotBlank(data.get(11))) {
map.put(city + region + data.get(7), data)
}
}
print(map.size())
Long count = 0;
for (List<Object> data : list) {
String city = data.get(0)
String region = data.get(1)
String primarySchool = data.get(7)
String key = city + region + primarySchool
if (map.get(key) != null && StringUtils.isBlank(data.get(11))) {
List<Object> val = map.get(key)
data.set(11, val.get(11))
data.set(12, val.get(12))
data.set(13, val.get(13))
data.set(14, val.get(14))
data.set(15, val.get(15))
count++
println(data)
}
}
print("优化数量" + count)
String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站",
"小区ID","数据类型"];
List<String> headerList = Arrays.asList(header)
list.add(0, headerList)
ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", list, "数据")
}
}
@@ -0,0 +1,67 @@
package com.fangjia.script.temp.shenjian.school
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ExcelUtils
/**
* 标准化文件输出
*
* @author shenjian
* @since 2021/4/16
*/
class Standard {
static void main(String[] args) {
List<List<Object>> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V2.0.xlsx")
def resultOne = []
def resultTwo = []
def resultThree = []
for (List<Object> data : list) {
// "320500","610100","340100"
if (data.get(0) == "320500") {
saveData(resultOne, data)
} else if (data.get(0) == "610000") {
data.set(0, 610100)
saveData(resultTwo, data)
} else if (data.get(0) == "340100") {
saveData(resultThree, data)
}
}
def headers = ["城市","区域","小区名称","小区经度","小区纬度", "小区地址","对口小学","小学地址","小学属性","小学批次","小学经度","小学纬度","对口中学","中学地址","中学属性",
"中学批次","中学经度","中学纬度","备注","创建时间","修改时间","来源网站"]
FileUtils.batchCreateFile(resultOne, "F:\\工作\\数据采购\\学校\\bigdata_commschool_320500.dat", headers, "utf-8")
FileUtils.batchCreateFile(resultTwo, "F:\\工作\\数据采购\\学校\\bigdata_commschool_610100.dat", headers, "utf-8")
FileUtils.batchCreateFile(resultThree, "F:\\工作\\数据采购\\学校\\bigdata_commschool_340100.dat", headers, "utf-8")
}
static void saveData(def result, def data) {
def ret = []
ret[0] = data.get(0)
ret[1] = data.get(1)
ret[2] = data.get(2)
ret[3] = data.get(3) != null && data.get(3) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(3))) : ""
ret[4] = data.get(4) != null && data.get(4) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(4))) : ""
ret[5] = data.get(5)
ret[6] = data.get(6)
ret[7] = data.get(7)
ret[8] = data.get(8)
ret[9] = ""
ret[10] = data.get(9) != null && data.get(9) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(9))) : ""
ret[11] = data.get(10) != null && data.get(10) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(10))) : ""
ret[12] = data.get(11)
ret[13] = data.get(12)
ret[14] = data.get(13)
ret[15] = ""
ret[16] = data.get(14) != null && data.get(14) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(14))) : ""
ret[17] = data.get(15) != null && data.get(15) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(15))) : ""
ret[18] = data.get(16)
ret[19] = data.get(17)
ret[20] = data.get(18)
ret[21] = data.get(19)
result << ret
}
}
@@ -0,0 +1,112 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区建筑结构
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class BuildStructure {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static Map buildStructureMap = new HashMap()
static {
buildStructureMap.put("钢结构", "3")
buildStructureMap.put("钢、钢筋混凝土结构", "1")
buildStructureMap.put("钢筋混凝土结构", "1")
buildStructureMap.put("钢混", "1")
buildStructureMap.put("钢混结构", "1")
buildStructureMap.put("混合结构", "11")
buildStructureMap.put("钢筋混凝土", "11")
buildStructureMap.put("砖木结构", "10")
buildStructureMap.put("其他结构", "11")
buildStructureMap.put("砖混结构", "2")
buildStructureMap.put("框架结构", "4")
buildStructureMap.put("框架剪力墙结构", "5")
buildStructureMap.put("剪力墙结构", "6")
buildStructureMap.put("筒体结构", "7")
buildStructureMap.put("框架筒体结构", "8")
buildStructureMap.put("筒中筒", "9")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("BuildStructure")
log.info("开始导出小区建筑结构表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("build_structure", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","建筑结构","备注", "创建时间", "更新时间", "来源公司"]
def seaweedCount = [21596, 17053, 5410];
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger count = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, count)
println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\建筑结构\\bigdata_commbuildstru_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区建筑结构表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << this.getBuildStructure(dbo.get("build_structure")) // 建筑结构
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
if (StringUtils.isNotBlank(dbo.get("build_structure"))) {
rows << row
count.incrementAndGet()
}
})
return rows
};
static def getBuildStructure(def houseStructure) {
def ret
if (StringUtils.isNotBlank(houseStructure)) {
ret = buildStructureMap.get(houseStructure)
if (StringUtils.isBlank(ret)) {
ret = "11" // 其他
}
} else {
ret = ""
}
return ret
}
}
@@ -0,0 +1,129 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区建筑类型表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class BuildType {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static Map buildTypeMap = new HashMap()
static {
buildTypeMap.put("多层", "0104")
buildTypeMap.put("小高层", "0103")
buildTypeMap.put("高层", "0199")
buildTypeMap.put("超高层", "0199")
buildTypeMap.put("花园洋房", "0199")
buildTypeMap.put("老洋房", "0199")
buildTypeMap.put("四合院", "0199")
buildTypeMap.put("独栋", "0105")
buildTypeMap.put("双拼", "0199")
buildTypeMap.put("联排", "0106")
buildTypeMap.put("叠加", "0107")
buildTypeMap.put("老式别墅洋房", "0199")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("BuildStructure")
log.info("开始导出小区建筑类型表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","建筑类型","备注", "创建时间", "更新时间", "来源公司"];
def seaweedCount = [21596, 17053, 5410];
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger count = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, count)
println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\建筑类型\\bigdata_commbuildtype_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区建筑类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
if (StringUtils.isNotBlank(dbo.get("build_type"))) {
Set set = this.getBuildTypeCode(dbo.get("build_type"))
for (String type : set) {
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << type // 建筑类型
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
rows << row
}
if (set.size() > 0) {
count.incrementAndGet()
}
}
})
return rows
}
static Set getBuildTypeCode(def buildType) {
Set<String> set = new HashSet<>()
if (StringUtils.isNotBlank(buildType)) {
String[] types = buildType.split(",")
for (String type : types) {
set.add(buildTypeMap.get(type))
}
if (set.isEmpty()) {
set.add("0199") // 其他
}
}
return set
}
static List getBuildTypeCodeSeq(def buildType) {
List list = new ArrayList()
if (StringUtils.isNotBlank(buildType)) {
String[] types = buildType.split(",")
for (String type : types) {
list.add(buildTypeMap.get(type))
}
if (list.isEmpty()) {
list.add("0199") // 其他
}
}
return list
}
}
@@ -0,0 +1,91 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.text.SimpleDateFormat
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区建筑年代表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class BuildYear {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("BuildYear")
log.info("开始导出小区建筑年代表")
def cites = ["北京", "重庆", "哈尔滨"]
def seaweedCount = [21596, 17053, 5410];
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("finish_building_date", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","建筑年代","备注", "创建时间", "更新时间", "来源公司"]
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger count = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, count)
println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\建筑年代\\bigdata_commbuildyear_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区建筑年代表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
Integer year = CommonUtils.getBuildYear(dbo.get("finish_building_date")) // 建筑年代
if (year > 1900) {
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << year
row << "" // 备注
Integer createYear = CommonUtils.getBuildYear(DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy"))
if (year > createYear) {
row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date());
row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date())
} else {
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
}
row << "fjw" // 来源公司
rows << row
count.incrementAndGet()
}
})
return rows
}
}
@@ -0,0 +1,87 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.core.util.ExcelUtil
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
/**
* @author shenjian
* @since 2021-03-13
*/
class CollectTest {
static void main(String[] args) {
def cites = ["北京", "重庆", "哈尔滨"];
for (def city : cites) {
/**
* _id 小区ID
* name 小区名
* alias 小区曾用名
*/
def districts = this.findAllCityDistricts(city, "district",
["city", "_id", "region", "name", "alias"])
print(districts)
}
}
static def findAllCityDistricts(def city, def cat, def fields) {
DBObject q = new BasicDBObject();
q.put("visible", 0)
q.put("cat", cat)
q.put("region", "密云")
if (city) {
q.put("city", city)
}
DBObject keys = new BasicDBObject();
if (fields) {
for (def field : fields) {
keys.put(field, 1)
}
}
DSCursor cursor = CommonDatautils.getDataStoreConnection().find(q, keys).batchSize(4000)
def districtKeys = []
def rows = [];
rows << ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积",
"小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"]
for (def dbo : cursor) {
def row = [];
row << null // TODO
row << dbo.get("_id")
row << dbo.get("name")
row << dbo.get("alias")
row << null // TODO
row << dbo.get("street_community") // 街道 TODO 拆分为街道门牌号
row << dbo.get("street_community") // 门牌号
rows << dbo.get("address")
rows << dbo.get("all_area")
rows << dbo.get("build_area")
rows << dbo.get("land_property")
ExcelUtils.write("d:\\数据\\小区表.xlsx", rows, "小区")
StringBuilder data = new StringBuilder()
Set<String> keySet = dbo.keySet()
keySet.forEach({key ->
if ("alias" == key) {
def value = dbo.get(key)
if (value != null) {
value = value[0]
}
data.append(value)
} else {
data.append(dbo.get(key))
}
data.append("|@|")
})
districtKeys << StringUtils.substringBeforeLast(data.toString(), "|@|")
};
return districtKeys;
}
}
@@ -0,0 +1,155 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang3.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区公司表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class Company {
private static ExecutorService executorService = Executors.newFixedThreadPool(2);
// 物业公司
static final DBCollection PROPERTY_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136")
// 开发商
static final DBCollection DEVELOP_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136")
static final propertyCompany = new HashMap()
static final developCompany = new HashMap()
static {
PROPERTY_COMPANY_COLLECTION.find().forEach({company ->
propertyCompany.put(company.get("company"), company.get("_id").toString())
})
DEVELOP_COMPANY_COLLECTION.find().forEach({company ->
developCompany.put(company.get("company"), company.get("_id").toString())
})
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("Company")
log.info("开始导出小区公司表")
def cites = ["北京", "重庆", "哈尔滨"]
def seaweedCount = [21596, 17053, 5410];
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("estate_company", 1).append("developer", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","公司ID","公司类型", "备注", "创建时间", "更新时间", "来源公司"]
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger developCount = new AtomicInteger(0)
AtomicInteger estateCount = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, developCount, estateCount)
println("城市" + city + " " + cityId + "物业数量为" + estateCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((estateCount.get() * 100) / seaweedCount[i])) )
println("城市" + city + " " + cityId + "开发商数量为" + developCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((developCount.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\公司\\bigdata_commcomp_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区公司表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger developCount, AtomicInteger estateCount) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
def rowCanSave = true
def row1CanSave = true
row << cityId // 城市ID
row << dbo.get("_id").toString()
def propertyCompany = dbo.get("estate_company")
if (StringUtils.isNotBlank(propertyCompany)) {
row << this.getPropertyCompanyId(propertyCompany) // 公司ID
row << "SV" // 公司类型(物业管理公司SV、 房地产开发商DV )
} else {
rowCanSave = false
}
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
def row1 = []
row1 << cityId
row1 << dbo.get("_id").toString()
def developerCompany = dbo.get("developer")
if (StringUtils.isNotBlank(developerCompany)) {
row1 << this.getDevelopCompanyId(developerCompany) // 公司ID
row1 << "DV" // 公司类型(物业管理公司SV、 房地产开发商DV )
} else {
row1CanSave = false
}
row1 << "" // 备注
row1 << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row1 << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row1 << "fjw" // 来源公司
if (rowCanSave) {
rows << row
estateCount.incrementAndGet()
}
if (row1CanSave) {
rows << row1
developCount.incrementAndGet()
}
})
return rows
}
/**
* 获取开发商公司ID
* @param company 公司名称
* @return
*/
static def getDevelopCompanyId(String company) {
if (StringUtils.isBlank(company)) {
return ""
}
return developCompany.get(company)
}
/**
* 获取物业公司ID
* @param company 公司名称
* @return
*/
static def getPropertyCompanyId(String company) {
if (StringUtils.isBlank(company)) {
return ""
}
return propertyCompany.get(company)
}
}
@@ -0,0 +1,36 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
/**
* @author shenjian
* @since 2021/6/4
*/
class CountSeaweed {
static void main(String[] args) {
def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"]
for (def city : cities) {
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("cat", "district");
q.put("visible", 0);
if (city == "福安") {
city = "宁德"
q.put("city", city)
q.put("region", "福安")
} else if (city == "平潭") {
city = "福州"
q.put("city", city)
q.put("region", "平潭")
} else if (city == "石狮") {
city = "泉州"
q.put("city", city)
q.put("region", "石狮")
}
println(city + " " + DbUtils.seaweed.count(q))
}
}
}
@@ -0,0 +1,112 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区费用表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class Fee {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("Fee")
log.info("开始导出小区费用表")
def cites = ["北京", "重庆", "哈尔滨"]
def seaweedCount = [21596, 17053, 5410];
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("estate_company", 1).append("estate_charge", 1)
.append("estate_type2", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID", "公司ID", "费用", "房屋用途","建筑类型", "小区费用类型", "费用说明", "备注", "创建时间", "更新时间", "来源公司"]
for (int i = 0; i < cites.size(); i++) {
def city = cites[i]
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
AtomicInteger count = new AtomicInteger(0)
def result = this.findAllCityDistricts(city, "district", cityId, filter, count)
println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) )
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\费用\\bigdata_commfee_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区费用表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def estate_charge = dbo.get("estate_charge")
if (estate_charge && estate_charge.toString().indexOf("~") != -1) {
String[] arr = estate_charge.toString().split("~")
estate_charge = ((Double.valueOf(arr[0]) + Double.valueOf(arr[1])) / 2.0D).toString()
}
if (estate_charge == '-' || estate_charge == '无') {
estate_charge = ""
}
if (StringUtils.isNotBlank(dbo.get("estate_company")) && StringUtils.isNotBlank(estate_charge)
&& StringUtils.isNotBlank(dbo.get("estate_type2")) && StringUtils.isNotBlank(dbo.get("build_type"))) {
List buildTypes = BuildType.getBuildTypeCodeSeq(dbo.get("build_type"))
Set<String> savedBuildTypeSet = new HashSet<>()
boolean isFirst = true
for (String buildType : buildTypes) {
if (savedBuildTypeSet.contains(buildType)) {
continue
}
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString() // 小区ID
row << Company.getPropertyCompanyId(dbo.get("estate_company")) // 公司ID
// 处理 db.getCollection('seaweed').find({'estate_charge': '3.3~4.3'}, {'estate_charge': 1})
// 特殊费用 3.3~4.3
if (isFirst) {
row << estate_charge // 费用
isFirst = false
count.incrementAndGet()
} else {
row << ""
break
}
row << HousePurpose.getHousePurposeCode(dbo.get("estate_type2")) // 房屋用途
row << buildType // 建筑类型
row << "1" // 小区费用类型 物业费1 电梯费2 垃圾费3
row << dbo.get("") // 费用说明
row << dbo.get("") // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
rows << row
savedBuildTypeSet.add(buildType)
}
}
})
return rows
}
}
@@ -0,0 +1,109 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.core.util.StringUtil
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区房屋用途表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class HousePurpose {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static Map housePurposeMap = new HashMap()
static {
housePurposeMap.put("普通住宅", "11")
housePurposeMap.put("别墅", "11")
housePurposeMap.put("商住", "13")
housePurposeMap.put("其他", "99")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("BuildStructure")
log.info("开始导出小区房屋用途表表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("estate_type2", 1).append("household_count", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","房屋用途","户数", "备注", "创建时间", "更新时间", "来源公司"];
for (def city : cites) {
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
def result = this.findAllCityDistricts(city, "district", cityId, filter)
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\房屋用途\\bigdata_commhouseuse_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区房屋用途表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << getHousePurposeCode(dbo.get("estate_type2"))
if (dbo.get("household_count") != null) {
if (Long.valueOf(dbo.get("household_count")) < 50) {
row << "" // 户数
} else {
row << dbo.get("household_count")
}
} else {
row << "" // 户数
}
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
if (dbo.get("m_date")) {
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
} else {
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss")
}
row << "fjw" // 来源公司
if (StringUtils.isNotBlank(dbo.get("estate_type2"))) {
rows << row
}
})
return rows
};
static def getHousePurposeCode(def housePurpose) {
def ret
if (StringUtils.isNotBlank(housePurpose)) {
ret = housePurposeMap.get(housePurpose)
if (StringUtils.isBlank(ret)) {
ret = "99" // 其他
}
} else {
ret = ""
}
return ret
}
}
@@ -0,0 +1,99 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区房屋类型表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class HouseType {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static Map houseTypeMap = new HashMap()
static {
houseTypeMap.put("普通住宅", "01")
houseTypeMap.put("商住", "07")
houseTypeMap.put("别墅", "08")
houseTypeMap.put("其他", "99")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("HouseType")
log.info("开始导出小区房屋类型表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("estate_type2", 1).append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","小区ID","小区房屋类型","备注", "创建时间", "更新时间", "来源公司"]
for (def city : cites) {
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
def result = this.findAllCityDistricts(city, "district", cityId, filter)
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\房屋类型\\bigdata_commhousetype_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区房屋类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << this.getHouseTypeCode(dbo.get("estate_type2")) // 小区房屋类型
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
if (dbo.get("m_date")) {
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
} else {
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss")
}
row << "fjw" // 来源公司
if (StringUtils.isNotBlank(dbo.get("estate_type2"))) {
rows << row
}
})
return rows
};
static def getHouseTypeCode(def houseType) {
def ret
if (StringUtils.isNotBlank(houseType)) {
ret = houseTypeMap.get(houseType) // 房屋类型
if (StringUtils.isBlank(ret)) {
ret = "99" // 其他
}
} else {
ret = ""
}
return ret
}
}
@@ -0,0 +1,70 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.statistics.seaweed.HouseDetail
import com.fangjia.script.statistics.seaweed.HouseList
import com.fangjia.script.statistics.seaweed.School
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
/**
* @author shenjian* @since 2021/9/2
*/
class NewHouseFangDi {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018)
public static Map<String, String> headerMap(String cook) {
Map<String, String> headers = new HashMap<String, String>();
headers.put("Host", "www.fangdi.com.cn")
headers.put("Connection", "keep-alive")
headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36")
headers.put("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")
headers.put("Accept", "application/json, text/javascript, */*; q=0.01")
headers.put("X-Requested-With", "XMLHttpRequest")
headers.put("Origin", "http://www.fangdi.com.cn")
headers.put("Referer", "http://www.fangdi.com.cn/new_house/new_house_list_fast.html?dicPriceID=&dicRegionID=&flatID=&districtID=5391be55e75e5506&check_input2=105C")
headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3")
headers.put("Accept-Encoding", "gzip, deflate")
headers.put("Accept-Language", "zh-CN,zh;q=0.9")
headers.put("Cookie", "www.fangdi.com_http_ic=www.fangdi.com.cn_80_RS; www.fangdi.com.cn=www.fangdi.com.cn_rs3; FSSBBIl1UgzbN7N80S=QS7tHXu4vqldPlDeqJkUCGoyqBmqHzTtyqvrk2ibcRKGVcbPOJ7ERuh5qYgHVD4l; www.fangdi.com_http_ic=www.fangdi.com.cn_80_RS; FSSBBIl1UgzbN7Nenable=true; JSESSIONID1=jw-mUJZsTiFYkXShzn6asMa401YoEZs-0WzhNHrUGId4Jh1_14tj!-338668155; FSSBBIl1UgzbN7N80T=4t2lc885DGECHVNGMwL4w1LcI9v43OwswK7ZOXM5ZbcyVF1UvDPUs59KVOQkq_2hxc0iJrZi3gBiildN1Xfh7xNz81qFz12KwlUJrncPg8mejTeav9tTOgiiPXMZZfJ4rltOCYgOzLLqbj5nG9esEse9UWPYnFdkTWvjpJUTfdyIGg9vB_S9ABAvmir3e89JTbzvfK4.i_IRPFED2hsteZcTiIpNWteOxZMMeDPSnKJei2_Hqhmn7LQMVZanibREAJ8Ah7IxIAwcMtRqAtZLLRpcQMtwnqkEoUPHf435yJVKL20fHui11nhVV4MxTHo9pcoz6L.yhuezCyP5NGTdpV14XV_yBCBhNkSXKWbO.lWAEUQ8yBkMwLLgaYY34sg0Bz_0")
return headers;
}
static void main(String[] args) {
Map<String, String> headers = headerMap("")
HttpRequest re = new HttpRequest();
String str = null
try {
Map<String, String> params = new HashMap<String, String>();
// params.put("districtID", "5391be55e75e5506")
// params.put("dicRegionID", "")
// params.put("dicPriceID", "")
// params.put("flatID", "")
params.put("districtID", "5391be55e75e5506")
params.put("dicRegionID", "")
params.put("stateID", "4d29bb4241910ce3")
params.put("houseAreaID", "0")
params.put("dicAvgpriceID", "0")
params.put("dicPositionID", "")
params.put("houseTypeID", "0")
params.put("houseTypeID", "0")
params.put("address", "")
params.put("openingID", "")
params.put("projectName", "")
params.put("currentPage", "1")
str = re.post("http://www.fangdi.com.cn/service/freshHouse/getFastHosueList.action", params, headers, "UTF-8");
} catch (Exception e) {
e.printStackTrace()
}
def data = JSONObject.parseObject(str).get("data")
print(data)
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区地理位置表
*
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class Position {
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("Location")
log.info("开始导出小区地理位置表结构表")
def cites = ["北京", "重庆", "哈尔滨"]
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("lng", 1).append("lat", 1)
.append("c_date", 1).append("m_date", 1)
def headers = ["城市ID","物体类型","物体ID","位置类型", "序号", "经度", "纬度", "海拔", "备注", "创建时间", "更新时间", "来源公司"]
for (def city : cites) {
def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID
def result = this.findAllCityDistricts(city, "district", cityId, filter)
executorService.execute(new Runnable() {
@Override
void run() {
FileUtils.batchCreateFile(result, "d:\\数据\\地理位置\\bigdata_position_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区地理位置表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter) {
def rows = []
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
def row = []
row << cityId // 城市ID
row << "5" // 物体类型 小区5
row << dbo.get("_id").toString() // 物体ID
row << "1" // 位置类型 中心点1
row << "1" // 序号
row << DataHandleUtils.getDoubleRound(6, dbo.get("lng") / 10000000000d) // 经度
row << DataHandleUtils.getDoubleRound(6, dbo.get("lat") / 10000000000d) // 纬度
row << "" // 海拔
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
rows << row
})
return rows
}
}
@@ -0,0 +1,288 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.temp.shenjian.dto.SearchAddressDTO
import com.fangjia.script.temp.shenjian.fill.CalcCubageAndVirescence
import com.fangjia.script.temp.shenjian.util.DistrictUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.springframework.util.StopWatch
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.regex.Pattern
/**
* @author shenjian
* @since 2021-03-13
*/
@Slf4j
class Seaweed {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
private static ExecutorService executorService = Executors.newFixedThreadPool(3);
private static HttpRequest http = new HttpRequest()
private static final String TOKEN = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
private static Map seaweedTypeMap = new HashMap();
private static Pattern pattern = Pattern.compile("\\s*|\t|\r|\n"); // 取消空格换行等
private static Map cityDistrictMap = new HashMap();
static {
seaweedTypeMap.put("普通住宅", "pa101")
seaweedTypeMap.put("别墅", "pa101")
seaweedTypeMap.put("商住", "pa101")
}
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("Seaweed")
log.info("开始导出小区公司表")
//def cites = ["北京", "重庆", "哈尔滨"]
//def seaweedCount = [21596, 17053, 5410];
def cites = ["哈尔滨"]
def seaweedCount = [5410];
DBObject filter = new BasicDBObject()
filter.append("_id", 1).append("name", 1).append("alias", 1).append("street_community", 1)
.append("address", 1).append("all_area", 1).append("build_area", 1).append("region", 1)
.append("estate_type2", 1).append("virescence_percent", 1).append("cubage_percent", 1)
.append("car_site_count", 1).append("c_date", 1).append("m_date", 1)
.append("lng", 1).append("lat", 1)
def headers = ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积",
"小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"]
for (int i = 0; i < cites.size(); i++) {
final t = i;
def city = cites[t]
def sameRegionDistricts = [:]
def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts)
BasicDBObject cityDistrictInfo = FangJiaAreaMapUtils.convertToOfficial(city)
def cityId = cityDistrictInfo.get("city_code") // 城市ID
final def map = this.findAllCityDistricts(city, "district", cityId, filter, sameRegionDistricts)
executorService.execute(new Runnable() {
@Override
void run() {
def result = []
AtomicInteger virescenceCount = new AtomicInteger(0) // 绿化率
AtomicInteger cubageCount = new AtomicInteger(0) // 容积率
AtomicInteger allAreaCount = new AtomicInteger(0)
AtomicInteger buildAreaCount = new AtomicInteger(0)
for (def key : map.keys()) {
result << map.get(key)
if (map.get(key) && map.get(key)[8] != "") {
allAreaCount.incrementAndGet()
}
if (map.get(key) && map.get(key)[9] != "") {
buildAreaCount.incrementAndGet()
}
if (map.get(key) && map.get(key)[12] != "") {
virescenceCount.incrementAndGet()
}
if (map.get(key) && map.get(key)[13] != "") {
cubageCount.incrementAndGet()
}
}
println("城市" + city + " " + cityId + "绿化率数量为" + virescenceCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((virescenceCount.get() * 100) / seaweedCount[t])) )
println("城市" + city + " " + cityId + "容积率数量为" + cubageCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((cubageCount.get() * 100) / seaweedCount[t])) )
println("城市" + city + " " + cityId + "占地面积数量为" + allAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((allAreaCount.get() * 100) / seaweedCount[t])) )
println("城市" + city + " " + cityId + "建筑面积数量为" + buildAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((buildAreaCount.get() * 100) / seaweedCount[t])) )
FileUtils.batchCreateFile(result, "d:\\数据\\小区\\bigdata_community_" + cityId + ".dat", headers, "UTF-8")
}
})
}
stopWatch.stop()
log.info("完成导出小区表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60)
executorService.shutdown()
fixedThreadPool.shutdown()
}
static def findAllCityDistricts(def city, def cat, def cityId, def filter, def sameRegionDistricts) {
def map = new ConcurrentHashMap();
DBObject query = new BasicDBObject()
query.append("cat", cat).append("visible", 0).append("city", city)
def total = DbUtils.seaweed.find(query, filter).count()
final CountDownLatch latch = new CountDownLatch(total)
DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo ->
fixedThreadPool.execute(new Runnable() {
@Override
void run() {
try {
dealSpecialChar(dbo)
def row = []
row << cityId // 城市ID
row << dbo.get("_id").toString()
row << dbo.get("name")
if (dbo.get("alias") != null && dbo.get("alias")[0] != null) {
row << pattern.matcher(dbo.get("alias")[0]).replaceAll("")
} else {
row << ""
}
def districtId
if (StringUtils.isBlank(cityDistrictMap.get(dbo.get("region")))) {
districtId = FangJiaAreaMapUtils.convertToOfficial(city, dbo.get("region")).get("district_code")
// 行政区ID
cityDistrictMap.put(dbo.get("region"), districtId)
} else {
districtId = cityDistrictMap.get(dbo.get("region"))
}
row << districtId
Map params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city);
params.put("address", dbo.get("address"));
// 暂时不用解析
// String s = http.get("http://open.fangjia.com/address/standardize", params, "utf-8");
// SearchAddressDTO searchAddress = com.alibaba.fastjson.JSONObject.parseObject(s, SearchAddressDTO.class)
// if (searchAddress.code == '200') {
// //row << searchAddress.result.street // 街道
// row << ""
// row << searchAddress.result.streetNumber // 门牌号
// } else {
// row << ""
// row << ""
// }
row << "" // 街道
row << "" // 门牌号
row << dbo.get("address")
if (dbo.get("all_area") && dbo.get("all_area").toString().indexOf("公顷") != -1) {
def val = dbo.get("all_area").toString()
def temp = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000).longValue()
if (temp > 100000) {
row << ""
} else {
row << temp
}
} else {
if (dbo.get("all_area")) {
if (Double.valueOf(dbo.get("all_area")) > 100000) {
row << ""
} else {
row << dbo.get("all_area")
}
} else {
row << ""
}
}
if (dbo.get("build_area") && Double.valueOf(dbo.get("build_area")) > 1000) {
row << dbo.get("build_area")
} else {
row << ""
}
row << "pa101" // 小区类型
row << "" // 是否新盘
if (dbo.get("virescence_percent") != null && dbo.get("cubage_percent") != null) {
// 绿化率
if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) < 1) {
row << Double.valueOf(dbo.get("virescence_percent")) * 100
} else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) == 1) {
row << Double.valueOf(dbo.get("virescence_percent")) * 10
} else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) >= 70) {
row << 65
} else {
row << Double.valueOf(dbo.get("virescence_percent"))
}
if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 7 && Double.valueOf(dbo.get("cubage_percent")) < 8)) {
row << 6.2
} else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 8 && Double.valueOf(dbo.get("cubage_percent")) < 9)) {
row << 6.3
} else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 9 && Double.valueOf(dbo.get("cubage_percent")) < 10)) {
row << 6.5
} else if (dbo.get("cubage_percent") && Double.valueOf(dbo.get("cubage_percent")) >= 10) {
row << 6.7
} else {
row << dbo.get("cubage_percent") // 容积率
}
} else {
def temp = CalcCubageAndVirescence.findResult(dbo.lat / 10000000000.0d, dbo.lng / 10000000000.0d, 500, sameRegionDistricts.get(dbo.region), dbo.name, dbo._id)
if (temp[2] && Double.valueOf(temp[2]) < 1) {
row << Double.valueOf(temp[2]) * 100
} else if (temp[2] && Double.valueOf(temp[2]) == 1) {
row << Double.valueOf(temp[2]) * 10
} else if (temp[2] && Double.valueOf(temp[2]) >= 70) {
row << 65
} else {
row << temp[2] // 绿化率
}
if (temp[1] && (Double.valueOf(temp[1]) >= 7 && Double.valueOf(temp[1]) < 8)) {
row << 6.2
} else if (temp[1] && (Double.valueOf(temp[1]) >= 8 && Double.valueOf(temp[1]) < 9)) {
row << 6.3
} else if (temp[1] && (Double.valueOf(temp[1]) >= 9 && Double.valueOf(temp[1]) < 10)) {
row << 6.5
} else if (temp[1] && Double.valueOf(temp[1]) >= 10) {
row << 6.7
} else {
row << temp[1] // 容积率
}
}
if (dbo.get("car_site_count") != null) {
if (Long.valueOf(dbo.get("car_site_count")) > 2000 || Long.valueOf(dbo.get("car_site_count")) < 100) {
row << ""
} else {
row << dbo.get("car_site_count") // 车位数量
}
} else {
row << "" // 车位数量
}
row << "" // 介绍
row << "" // 备注
row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间
row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间
row << "fjw" // 来源公司
map.put(dbo.get("_id").toString(), row)
} catch(Exception e) {
} finally {
latch.countDown()
}
}
})
})
latch.await()
return map
};
/**
* 处理特殊字符
* @param dbo
*/
static def dealSpecialChar(DBObject dbo) {
Set<String> keySet = dbo.keySet()
for (String key : keySet) {
if (String.isInstance(dbo.get(key))) {
dbo.put(key, pattern.matcher(dbo.get(key)).replaceAll(""))
}
}
};
static def getSeaweedTypeCode(def value) {
def ret
if (StringUtils.isNotBlank(value)) {
ret = seaweedTypeMap.get(value)
if (StringUtils.isBlank(ret)) {
ret = "99" // 其他
}
} else {
ret = ""
}
return ret
}
}
@@ -0,0 +1,63 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
/**
* 小区别名数量统计
*
* @author shenjian
* @since 2021/4/15
*/
class SeaweedAliasExport {
static void main(String[] args) {
def cities = ["重庆","苏州","昆明","柳州","曲靖","三明","达州","石狮","大理","福安","平潭","迪庆州"]
def result = []
result << ["城市", "小区数量", "别名数量", "别名覆盖率"]
for (def city : cities) {
def ret = []
DBObject query = new BasicDBObject();
query.put("city", city)
query.put("cat", "district")
query.put("visible", 0)
//def seaweedCount = DBUtils.seaweed.find(query).batchSize(5000).count()
int seaweedCount = 0;
int aliasCount = 0;
if (city == "平潭") {
query.put("city", "福州")
query.put("region", city)
}
if (city == "福安") {
query.put("city", "宁德")
query.put("region", city)
}
if (city == "石狮") {
query.put("city", "泉州")
query.put("region", city)
}
DBUtils.seaweed.find(query).batchSize(5000).each {obj ->
if (obj.alias) {
aliasCount++
}
seaweedCount++
}
ret[0] = city
ret[1] = seaweedCount
ret[2] = aliasCount
if (seaweedCount > 0) {
ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(aliasCount) * 100 / Double.valueOf(seaweedCount))
}
result << ret
}
println(result)
ExcelUtils.write("F:\\数据\\小区别名覆盖率.xlsx", result, "统计")
}
}
@@ -0,0 +1,51 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
/**
* 小区geohash导出
*
* @author shenjian
* @since 2021/5/25
*/
class SeaweedGeoHashExport {
static void main(String[] args) {
print(SeaweedUtils.encodeByGeoHash(30.8006287800, 108.3835777247, 8))
//List<List<String>> data = com.fangjia.script.util.ExcelUtils.read("D:\\Users\\Domi\\Desktop\\房价网\\项目\\银联\\长宁区融合数据结果20210625.xlsx")
/* for (List<String> str : data) {
String geohash = str.get(0)
str.add(SeaweedUtils.decodeByGeoHash(geohash))
}*/
//com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\银联\\长宁区融合数据结果20210625.xlsx", data, "geohash");
// DBObject query = new BasicDBObject()
// query.append("cat", "district").append("visible", 0).append("city", "上海").append("region", "长宁")
//
// DBObject filter = new BasicDBObject()
// filter.put("lng", 1)
// filter.put("lat", 1)
// filter.put("name", 1)
//
// def rows = []
// DbUtils.seaweed.find(query, filter).batchSize(20000).each {objTemp ->
// final def obj = objTemp
// if (obj.lng && obj.lat) {
// def lon = obj.lng
// def lat = obj.lat
// def geohash = SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)
// rows << [lon, lat, obj.name, geohash]
// }
// }
// def ret = []
// ret << ["经度", "纬度", "小区名", "小区geohash"]
// ret.addAll(rows)
// com.fangjia.script.util.ExcelUtils.write("D:\\数据\\基础建设\\geohash.xlsx", ret, "geohash");
}
}
@@ -0,0 +1,195 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.Constants
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.statistics.seaweed.HouseDetail
import com.fangjia.script.statistics.seaweed.HouseList
import com.fangjia.script.statistics.seaweed.School
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.MongoCollection
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.jdom.Content;
/**
* 对口小学、初中
* @author shenjian
* @since 2021/9/2
*/
class SeaweedSchool {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018)
public static Map<String, String> headerMap(String cook) {
Map<String, String> headers = new HashMap<String, String>();
headers.put("Host", "www.mapmapping.com")
headers.put("Connection", "keep-alive")
headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("Content-Type", "application/json")
headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3")
headers.put("Referer", "https://servicewechat.com/wx5568ec264e167aeb/9/page-frame.html")
headers.put("Accept-Encoding", "gzip, deflate, br")
return headers;
}
static void main(String[] args) {
Map<String, String> headers = headerMap("")
HttpRequest re = new HttpRequest();
String str = null
//final String area = "浦东";
final int perPage = 10
final int total = 26766;
for (int page = 474; page < total / perPage; page++) {
print("当前页:" + page)
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", "上海市")
//params.put("area", area)
params.put("page", String.valueOf(page))
params.put("perPage", String.valueOf(perPage))
params.put("priceType", "")
params.put("order", "")
params.put("source", "1")
str = re.get("https://mapmapping.com/school-web/house/getHouseInfoList", params, headers, "UTF-8");
} catch (Exception e) {
e.printStackTrace()
}
println str
if (StringUtils.isNotEmpty(str)) {
HouseList houseList = JSONObject.parseObject(str, HouseList.class);
if (houseList != null && "000".equals(houseList.getCode())) {
/*Integer total = houseList.getContent().getTotal();*/
List<HouseList.ContentDTO.ListDTO> listDTOS = houseList.getContent().getList();
if (CollectionUtils.isNotEmpty(listDTOS)) {
for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) {
Integer id = listDTO.getId();
//# 房屋详情
HouseDetail houseDetail = getHouseDetail(re, id, headers)
// 对口幼儿园
School nurserySchool = getNurserySchool(re, id, headers)
//# 对口小学
School primarySchool = getPrimarySchool(re, id, headers)
//# 对口中学
School middleSchool = getMiddleSchool(re, id, headers)
String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id)
saveData(houseDetail, nurserySchool, primarySchool, middleSchool, url);
}
}
}
}
}
}
private static void saveData(HouseDetail houseDetail, School nurserySchool, School primarySchool, School middleSchool, def url) {
DBObject dbObject = new BasicDBObject()
dbObject.put("url", url)
dbObject.put("root", "沪上学区房")
dbObject.put("c_date", new Date())
dbObject.put("city", "上海")
dbObject.put("region", houseDetail.getContent().getArea())
dbObject.put("name", houseDetail.getContent().getName())
dbObject.put("address", houseDetail.getContent().getAddress())
dbObject.put("lng", houseDetail.getContent().getLng())
dbObject.put("lat", houseDetail.getContent().getLat())
List<School.ContentDTO.ListDTO> nurserySchools = nurserySchool.getContent().getList();
if (CollectionUtils.isNotEmpty(nurserySchools)) {
List<DBObject> schools = buildSchool(nurserySchools)
dbObject.put("nurserySchool", schools)
}
List<School.ContentDTO.ListDTO> primarySchools = primarySchool.getContent().getList();;
if (CollectionUtils.isNotEmpty(primarySchools)) {
List<DBObject> schools = buildSchool(primarySchools)
dbObject.put("primarySchool", schools)
}
List<School.ContentDTO.ListDTO> middleSchools = middleSchool.getContent().getList();;
if (CollectionUtils.isNotEmpty(middleSchools)) {
List<DBObject> schools = buildSchool(middleSchools)
dbObject.put("middleSchool", schools)
}
collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false)
}
private static List<DBObject> buildSchool(List<School.ContentDTO.ListDTO> currSchools) {
List<DBObject> schools = new ArrayList<>()
for (School.ContentDTO.ListDTO contentDTO : currSchools) {
DBObject schoolObject = new BasicDBObject();
if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(contentDTO.getTags())) {
if (contentDTO.getTags().size() >= 3) {
schoolObject.put("quality", contentDTO.getTags().get(0))
schoolObject.put("type", contentDTO.getTags().get(1))
schoolObject.put("batch", contentDTO.getTags().get(2))
} else if (contentDTO.getTags().size() == 2) {
schoolObject.put("quality", contentDTO.getTags().get(0))
schoolObject.put("type", contentDTO.getTags().get(1))
schoolObject.put("batch", '')
} else {
schoolObject.put("quality", contentDTO.getTags().get(0))
schoolObject.put("type", '')
schoolObject.put("batch", '')
}
}
schoolObject.put("area", contentDTO.getArea())
schoolObject.put("town", contentDTO.getTown())
schoolObject.put("distance", contentDTO.getDistance())
schoolObject.put("address", contentDTO.getAddress())
schoolObject.put("name", contentDTO.getName())
schools.add(schoolObject)
}
return schools
}
private static HouseDetail getHouseDetail(def re, def id, def headers) {
Map<String, String> params = new HashMap<String, String>();
params.put("id", String.valueOf(id))
params.put("source", "1")
def str = re.get("https://mapmapping.com/school-web/house/getHouseInfoDetail", params, headers, "UTF-8");
return JSONObject.parseObject(str, HouseDetail.class)
}
private static School getNurserySchool(def re, def id, def headers) {
Map<String, String> params = new HashMap<String, String>();
params.put("id", String.valueOf(id))
params.put("type", "1")
params.put("page", "1")
params.put("perPage", "2")
params.put("source", "1")
def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8");
return JSONObject.parseObject(str, School.class)
}
private static School getPrimarySchool(def re, def id, def headers) {
Map<String, String> params = new HashMap<String, String>();
params.put("id", String.valueOf(id))
params.put("type", "2")
params.put("page", "1")
params.put("perPage", "2")
params.put("source", "1")
def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8");
return JSONObject.parseObject(str, School.class)
}
private static School getMiddleSchool(def re, def id, def headers) {
Map<String, String> params = new HashMap<String, String>();
params.put("id", String.valueOf(id))
params.put("type", "3")
params.put("page", "1")
params.put("perPage", "4")
params.put("source", "1")
def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8");
return JSONObject.parseObject(str, School.class)
}
}
@@ -0,0 +1,19 @@
package com.fangjia.script.temp.shenjian.util
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
/**
* @author shenjian
* @since 2021/6/2
*/
class CityUtils {
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static listCitySet() {
List<String> list = cityMapConn.distinct("fangjia_city")
Set<String> citySet = new HashSet<>(list);
return citySet
}
}
@@ -0,0 +1,55 @@
package com.fangjia.script.temp.shenjian.util
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
/**
* ÇøÓò¹¤¾ßÀà
*/
class DistrictUtils {
static main(args) {
}
static def getAllDistricts(def cat, def city, def sameRegionDistricts){
DBObject query = new BasicDBObject();
query.append("cat", cat).append("visible", 0).append("city", city)
DBObject keys = new BasicDBObject().append("city", 1).append("region", 1).append("cubage_percent", 1)
.append("virescence_percent", 1).append("lat2", 1).append("lng2", 1)
.append("finish_building_date", 1).append("name", 1).append("_id", 1);
DSCursor cursor = DbUtils.seaweed.find(query, keys).batchSize(10000)
Iterator<DBObject> iters = cursor.iterator()
def result = []
while(iters.hasNext()) {
DBObject obj = iters.next()
def map = [:];
map.city = obj.city
map.cubage_percent = obj.cubage_percent;
map.virescence_percent = obj.virescence_percent;
map.lat2 = obj.lat2
map.lng2 = obj.lng2
map.region = obj.region
map.finish_building_date = obj.finish_building_date
map.name = obj.name
map._id = obj._id
result << map;
if (sameRegionDistricts.get(obj.region)) {
def temp = sameRegionDistricts.get(obj.region)
temp << map
sameRegionDistricts.put(obj.region, temp)
} else {
def temp = []
temp << map
sameRegionDistricts.put(obj.region, temp)
}
}
return result;
}
}
@@ -0,0 +1,113 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.excel.EasyExcel
import com.alibaba.excel.EasyExcelFactory
import com.alibaba.excel.ExcelReader
import com.alibaba.excel.context.AnalysisContext
import com.alibaba.excel.event.AnalysisEventListener
import com.alibaba.excel.read.metadata.ReadSheet
import java.util.concurrent.ConcurrentHashMap
/**
* excel工具类
*
* @author shenjian
* @since 2021/5/15
*/
class ExcelUtils {
static List<List<String>> formatHead(def headArr) {
List<List<String>> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
}
return head;
}
static List<List<String>> read(String filePath) {
SimpleExcelListener listener = new SimpleExcelListener(filePath);
ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(1).build();
excelReader.readAll()
List<List<String>> dataList = listener.getDataList();
excelReader.finish()
return dataList;
}
static List<List<String>> readSheet(String fileName, String[] sheetNames) {
Map<String, List<List<String>>> result = new ConcurrentHashMap<>();
ExcelReader excelReader = EasyExcel.read(fileName).build();
// 每个sheet分别创建监听器
SimpleExcelListener listener0 = new SimpleExcelListener();
SimpleExcelListener listener1 = new SimpleExcelListener();
ReadSheet readSheet0 = EasyExcel.readSheet(0).registerReadListener(listener0).build();
ReadSheet readSheet1 = EasyExcel.readSheet(1).registerReadListener(listener1).build();
// 这里注意 一定要把sheet1 sheet2 一起传进去,不然有个问题就是03版的excel 会读取多次,浪费性能
// 可以传多个ReadSheet对象,读多个sheet
excelReader.read(readSheet0, readSheet1);
// 这里千万别忘记关闭,读的时候会创建临时文件,到时磁盘会崩的
excelReader.finish();
ArrayList<SimpleExcelListener> listenerList = new ArrayList<>();
listenerList.add(listener0);
listenerList.add(listener1);
for (SimpleExcelListener listener : listenerList) {
result.put(listener.getSheetName(), listener.getDataList())
}
return result
}
public static List<List<String>> read(InputStream inputStream) {
SimpleExcelListener listener = new SimpleExcelListener();
ExcelReader excelReader = EasyExcelFactory.read(inputStream, null, listener)
.headRowNumber(1).build();
excelReader.readAll();
List<List<String>> dataList = listener.getDataList();
excelReader.finish();
return dataList;
}
public static class SimpleExcelListener extends AnalysisEventListener {
private List<List<String>> dataList = new ArrayList<>();
private String sheetName;
public SimpleExcelListener() {
}
public SimpleExcelListener(String sheetName) {
this.sheetName = sheetName
}
@Override
public void invoke(Object o, AnalysisContext analysisContext) {
Map<String, String> map = (Map<String, String>) o;
dataList.add(new ArrayList<>(map.values()));
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
List<List<String>> getDataList() {
return dataList;
}
void setDataList(List<List<String>> dataList) {
this.dataList = dataList;
}
public String getSheetName() {
return sheetName
}
}
}
@@ -0,0 +1,379 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONArray
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.CosineSimilarAlgorithm
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import org.apache.commons.lang.StringUtils
import java.util.concurrent.atomic.AtomicInteger
/**
* 高德工具类
*
* @author shenjian
* @since 2021/5/15
*/
class GaoDeUtils {
// 新获取key
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"]
def static AtomicInteger keyUsed = new AtomicInteger(0)
def static codeNameMap = new HashMap<String, String>(){{
put("150500", "地铁");
put("1507", "公交");
put("090100", "医院")
put("090101", "医院")
put("090200", "医院")
put("090202", "医院")
put("090202", "医院")
put("090204", "医院")
put("090205", "医院")
put("090206", "医院")
put("090206", "医院")
put("090208", "医院")
put("090209", "医院")
put("090210", "医院")
put("090211", "医院")
put("141200", "学校")
put("141201", "学校")
put("141202", "学校")
put("141203", "学校")
put("141204", "学校")
put("1601", "银行")
put("1603", "银行")
put("0601", "商场")
}};
/**
* 查找周边配套
* @param city
* @param kw
* @param lng
* @param lat
* @param radius
* @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw)
* @return
*/
static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){
def params = [:]
params.key = GAO_DE_AK[keyUsed]
if (types) {
params.types = types
} else {
params.keywords = keywords
}
params.city = city
params.radius = String.valueOf(radius)
params.location = "${lng},${lat}".toString()
List result = []
int page = 1
int count = 10000
while (result.size() < count) {
params.page = "${page}".toString()
page++
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
result.addAll(DataUtil.toMongoDbObj(json.pois))
} else if (json && json.infocode && json.infocode == 10003) {
throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
} else if (json && json.infocode && json.infocode == 10004) {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else {
break
}
}
return result
}
/**
* 查找周边配套
* @param city
* @param kw
* @param lng
* @param lat
* @param radius
* @param types
* @return
*/
static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){
def params = [:]
params.key = GAO_DE_AK[keyUsed.intValue()]
if (types) {
params.types = types
} else {
params.keywords = keywords
}
params.city = city
params.radius = String.valueOf(radius)
params.location = "${lng},${lat}".toString()
List result = []
int page = 1
int count = 10000
while (result.size() < count) {
params.page = "${page}".toString()
page++
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
try {
List<AmapPoi> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
result.addAll(ret)
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
break;
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
} else {
break
}
}
return result
}
/**
* 查找最可能的地址经纬度
*
* @param city
* @param address
* @param region
* @param typecodes,如130106|130105|130104
* @return
*/
static List<Double> getLngLat(def city, def address, def region, def name, def typecodes) {
List<AmapPoi> amapPoi = GaoDeUtils.searchPoi(city, address, region, typecodes)
if (amapPoi && amapPoi.size() > 0) {
def poi = amapPoi[0]
double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, name)
if (similarity < 0.7) {
amapPoi = GaoDeUtils.searchPoi(city, address, region, "")
}
poi = amapPoi[0]
String[] locations = poi.location.split(",")
def lng = Double.valueOf(locations[0])
def lat = Double.valueOf(locations[1])
return Arrays.asList(lng, lat)
} else {
amapPoi = GaoDeUtils.searchPoi(city, address, region, "")
if (amapPoi && amapPoi.size() > 0) {
def poi = amapPoi[0]
String[] locations = poi.location.split(",")
def lng = Double.valueOf(locations[0])
def lat = Double.valueOf(locations[1])
return Arrays.asList(lng, lat)
}
}
return Collections.emptyList();
}
/**
* 获取详情
*/
static List<AmapPoi> getAmapById(id){
def params = [:]
params.key = GAO_DE_AK[keyUsed.intValue()]
params.id = id
params.output = 'json'
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoi> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
return Collections.emptyList()
}
/**
* 查找关键字经纬度
*/
static List searchPoi(String city, String keywords, String region, String typecodes){
def params = [:]
params.key = GAO_DE_AK[keyUsed.intValue()]
params.keywords = keywords
params.city = city
params.region = region
if (StringUtils.isNotBlank(typecodes)) {
params.types = typecodes
}
params.put("extensions", "all")
params.put("citylimit","true")
params.put("offset","20")
List result = []
int page = 1
int count = 10000
while (result.size() < count) {
params.page = "${page}".toString()
page++
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
try {
List<AmapPoi> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
result.addAll(ret)
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
break;
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
}
return result
}
/**
* 获取经纬度之间距离
*/
static def getDistance(double Lat, double Lng, double Lat1, double Lng1) {
return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1)
}
/**
* 经纬度转地址<br>
* 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位
* @author yinjihuan
* @date 2016年9月9日
* @param location
* @return
*/
static def getGaoDeAddress(location) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK[keyUsed.intValue()])
params.put("location", location)
String content
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
}
net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content)
Map<String, Object> infoMap = DataHandleUtils.JsonToMap(json);
if (infoMap && infoMap.info == "OK") {
def regeocode = infoMap.regeocode
if (regeocode) {
return regeocode.formatted_address;
}
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
return null
}
/**
* 获取地址经纬度
*/
static def parserByGaode(def city, def address) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK[keyUsed.intValue()])
params.put("city", city)
params.put("address", address)
String content
try {
content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8")
}
net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content)
if (json && json.info && json.info == "OK") {
def ret = json.geocodes
if (ret) {
def str = ret.get(0).location
def level = ret.get(0).level
def arrs = str.split(",")
return ["lng": arrs[0], "lat": arrs[1]]
}
} else if (json && json.infocode && json.infocode == "10044") {
System.exit(1)
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.temp.shenjian.util
import org.apache.commons.httpclient.HttpClient
import org.apache.commons.httpclient.methods.PostMethod
import org.apache.commons.httpclient.methods.StringRequestEntity
/**
*
* @author shenjian
* @since 2021/5/25
*/
class HttpUtils {
static boolean postForJson(String url, String jsonString) {
HttpClient httpClient = new HttpClient()
StringRequestEntity requestEntity = new StringRequestEntity(
jsonString,
"application/json",
"UTF-8");
PostMethod postMethod = new PostMethod(url)
postMethod.setRequestEntity(requestEntity);
int statusCode = httpClient.executeMethod(postMethod);
if (statusCode == 200) {
return true
}
return false
}
}
@@ -0,0 +1,45 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSON
import org.msgpack.MessagePack
import org.msgpack.type.Value;
/**
* msg½âÎö¹¤¾ßÀà
*/
public class MessageUtils {
public static String unpack(byte[] content) {
MessagePack messagePack = new MessagePack();
try {
Value value = messagePack.read(content);
return value.toString();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
public static byte[] pack(String value){
MessagePack messagePack = new MessagePack();
try {
byte[] data = messagePack.write(value)
return data;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
public static byte[] pack(JSON json){
MessagePack messagePack = new MessagePack();
try {
byte[] data = messagePack.write(json)
return data;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
}
@@ -0,0 +1,40 @@
package com.fangjia.script.temp.shenjian.util
/**
* @author shenjian* @since 2021/7/7
*/
public class MyStringUtils {
public static joinSql(String[] str) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < str.size(); i++) {
sb.append("'").append(str[i]).append("'")
if (i != str.size() - 1) {
sb.append(",")
}
}
return sb.toString()
}
public static joinInSql(String[] str) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < str.size(); i++) {
sb.append("'").append(str[i]).append("'")
if (i != str.size() - 1) {
sb.append(",")
}
}
return sb.toString()
}
public static joinLikeSql(String[] str) {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < str.size(); i++) {
sb.append(" typecode like '").append(str[i]).append("%' ")
if (i != str.size() - 1) {
sb.append(" or ")
}
}
return sb.toString()
}
}
@@ -0,0 +1,209 @@
package com.fangjia.script.temp.shenjian.util
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.datastore.FieldValidatorUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.geometry.utils.Geohash
import java.util.concurrent.ConcurrentHashMap
/**
* 小区工具类
*
* @author shenjian
* @since 2021/5/17
*/
class SeaweedUtils {
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static def map = new ConcurrentHashMap()
static getRegionByCode(def district_code) {
def region;
if (map.get("district_code") != null) {
region = map.get("district_code")
} else {
cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj ->
region = obj.fangjia_district
map.put("district_code", region)
}
}
return region
}
/**
* 标准化输出绿化率
*/
static formatVirescencePercent(def virescence_percent) {
// 绿化率
if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) {
double val = Double.valueOf(virescence_percent)
if (val < 1) {
return val * 100
}
if (val == 1) {
return val * 10
}
if (val >= 70) {
return 65
}
return val
}
return ""
}
/**
* 标准化容积率输出
*/
static formatCubagePercent(def cubage_percent) {
if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) {
double val = Double.valueOf(cubage_percent)
if (val >= 7 && val < 8) {
return 6.2
}
if (val >= 8 && val < 9) {
return 6.3
}
if (val >= 9 && val < 10) {
return 6.5
}
if (val >= 10) {
return 6.7
}
return val
}
return ""
}
/**
* 标准化车位输出
*/
static formatCarSiteCount(def car_site_count) {
if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) {
long val = Long.valueOf(car_site_count)
if (val > 2000 || val < 100) {
return ""
}
return val;
}
return ""
}
/**
* 标准化建筑面积输出
*/
static formatBuildArea(def build_area) {
if (build_area && FieldValidatorUtils.checkValue(build_area)) {
double val = build_area
if (val > 1000) {
return val
}
}
return ""
}
/**
* 标准化总占地面积输出
*/
static formatAllArea(def all_area) {
if (all_area) {
Double formattedVal = 0.0;
if (all_area.toString().indexOf("公顷") != -1) {
def val = all_area.toString()
formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000)
} else if(FieldValidatorUtils.checkValue(all_area)) {
formattedVal = Double.valueOf(all_area)
}
if (formattedVal > 100000 || formattedVal == 0.0) {
return ""
}
return formattedVal
}
}
/**
* 标准化户数输出
*/
static formatHouseholdCount(def household_count) {
if (household_count && FieldValidatorUtils.checkValue(household_count) ) {
if (Long.valueOf(household_count) < 50) {
return "" // 户数
}
return household_count
}
}
/**
* 计算车位比
*/
static calcCarAndHouseholdRadio(def car_site_count, def household_count) {
if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) {
long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count)
long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count)
double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount)))
if (val == 0.0) {
val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount)))
}
if (val == 0.0) {
val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount)))
}
if (val == 1.0) {
return "1:1"
}
return "1:" + val
}
return ""
}
/**
* 标准化别名输出
*/
static formatAlias(def alias) {
if (alias) {
// 小区可能存在多个别名
def temp = []
int len = 0
for (def a : alias) {
len += a.toString().length()
if (len > 20) {
return temp.join(",")
}
temp.add(a)
}
return temp.join(",")
}
return ""
}
/**
* 格式化经纬度
*/
static def formatLatLng(def latOrLng) {
// 107.109878,29.192381
return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d)
}
/**
* 小区经纬度geohash编码
* @param args
*/
static def encodeByGeoHash(double lon, double lat, int level=7) {
return Geohash.stringEncode(lon, lat, level)
}
/**
* 小区经纬度geohash解码
* @param args
*/
static def decodeByGeoHash(String geohash) {
return DataHandleUtils.getDoubleRound(6, Geohash.decodeLatitude(geohash)) + "," +
DataHandleUtils.getDoubleRound(6, Geohash.decodeLongitude(geohash))
}
static void main(String[] args) {
def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"]
println(formatAlias(alias))
}
}
@@ -167,7 +167,16 @@ public class CommonUtils {
}
static void sendMessage(String queueName, byte[] bytes){
ConnectionFactory factory = new ConnectionFactory();
factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f");
Connection conn = factory.newConnection();
Channel channel = conn.createChannel();
channel.queueDeclare(queueName, true, false, false, null);
channel.basicPublish("", queueName, null, bytes);
channel.close()
conn.close()
}
/**
@@ -1,12 +1,15 @@
package com.fangjia.script.util
import com.fangjia.script.lixue.util.DateFormat
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
import org.apache.poi.hssf.record.BoolErrRecord
import org.apache.poi.hssf.usermodel.HSSFBorderFormatting
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.apache.poi.ss.formula.eval.ErrorEval
import org.apache.poi.ss.usermodel.*
import org.apache.poi.ss.util.CellRangeAddress
import org.apache.poi.ss.util.RegionUtil
import org.apache.poi.xssf.streaming.SXSSFWorkbook
import java.util.concurrent.ExecutorService
@@ -83,9 +86,6 @@ public class ExcelUtils{
}
}
ret.add(rowData);
if (ret.size() % 10000 == 0) {
println(ret.size())
}
}
} catch (IOException e) {
e.printStackTrace();
@@ -240,7 +240,7 @@ public class ExcelUtils{
return super.getProperty(propertyName)
}
public static void write(String fileName, List<List<Object>> data, String sheetName){
public static void write(String fileName, List<List<Object>> data, String sheetName, List<List<Integer>> mergeData){
File file = new File(fileName)
File dir = file.getParentFile();
@@ -309,6 +309,8 @@ public class ExcelUtils{
throw e
}
}
mergeCell(workBook, mainSheet, mergeData)
for(int i = 0; i < maxColumnNum; i++ ){
mainSheet.autoSizeColumn(i)
}
@@ -326,6 +328,36 @@ public class ExcelUtils{
fos.close();
}
static void write(String fileName, List<List<Object>> data, String sheetName){
write(fileName, data, sheetName, null)
}
/**
* 合并单元格
* @param workBook
* @param sheet
* @param data int firstRow, int lastRow, int firstCol, int lastCol
*/
static mergeCell(Workbook workBook, Sheet sheet, List<List<Integer>> data) {
if (CollectionUtils.isEmpty(data)) {
return
}
//设置style
CellStyle mergedCellStyle = workBook.createCellStyle()
mergedCellStyle.verticalAlignment = VerticalAlignment.CENTER;
mergedCellStyle.alignment = HorizontalAlignment.CENTER;
for (List<Integer> rowCell : data) {
if (rowCell.get(0) != rowCell.get(1)) {
CellRangeAddress cra = new CellRangeAddress(rowCell.get(0), rowCell.get(1), rowCell.get(2), rowCell.get(3))
sheet.addMergedRegion(cra);
//设置合并后style
def cell = sheet.getRow(rowCell.get(0)).getCell(rowCell.get(2));
cell.setCellStyle(mergedCellStyle)
}
}
}
public static newWorkBook(String fileName){
Workbook wb;
if(fileName.endsWith(".xlsx")){
@@ -179,31 +179,33 @@ public class KeywordSearchUtils{
public static final Map<String,String> KEYWORDS_MAP = new HashMap<String, String>(){{
put("地铁", "150500");
put("公交", "1507");
put("码头", "1503");
put("机场", "150104|150106");
put("火车站", "150200");
put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
put("写字楼", "120201");
put("停车场", "150903|150904|150905|150906");
put("网吧", "080308");
put("学校","141200|141201|141202|141203|141204");
put("银行","1601|1603");
put("公园/绿地","110100|110101");
put("KTV","080302");
put("健身中心","080111");
put("大型超市","0604");
put("咖啡厅","0505");
put("剧场/影院","080601|080603");
put("商场","0601");
put("酒店","10");
put("整容整形","090201");
put("洗浴中兴","0714");
put("便利店","0602");
put("餐饮","0501|0502|0503|0504");
put("游乐场","080501");
put("旅游景点","110000");
put("体育场馆","0800|0801");
put("图书馆","140500");
// put("码头", "1503");1
// put("机场", "150104|150106");1
// put("火车站", "150200");1
// put("停车场", "150903|150904|150905|150906");1
// put("网吧", "080308");1
// put("公园/绿地","110100|110101");
// put("KTV","080302");1
// put("健身中心","080111");1
// put("咖啡厅","0505");1
// put("剧场/影院","080601|080603");1
// put("酒店","10");1
// put("整容整形","090201");1
// put("洗浴中兴","0714");1
// put("便利店","0602");1
// put("游乐场","080501");1
// put("体育场馆","0800|0801");1
// put("住宅区","120300|120301|120302|120303|120304");
}};
@@ -0,0 +1,379 @@
package com.fangjia.script.statistics;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import com.alibaba.fastjson.JSONObject;
/**
* 蚂蚁分工开放平台工具类
* 由于groovy调用会误报签名错误问题,故用java方式
*/
public class ApiUtil {
private static final String secretKey = "HtRc7R5jI0plOOAukES4pM6yxWJbU9GyyM1xi4kH";
private static final String corpId = "ding0243a5c51f1f4aa8";
public static String getApiSingnature(String body, Long timeStamp, String secretKey) throws Exception {
JSONObject bodyObj = JSONObject.parseObject(body);
return getApiSingnature(bodyObj, timeStamp, secretKey);
}
/**
* 获取签名
*/
public static String getApiSingnature(JSONObject body, Long timeStamp, String secretKey) throws Exception {
String signature = "";
try {
//参数排序,保证签名一致
List<String> fileds = new ArrayList<>();
Set<Entry<String, Object>> set = body.entrySet();
for (Entry<String, Object> entry : set) {
fileds.add(entry.getKey());
}
//参数执行排序
Collections.sort(fileds);
Map<String, Object> params = new LinkedHashMap<String, Object>();
for (String key : fileds) {
Object obj = body.get(key);
if(obj != null){
params.put(key, obj);
}
}
String stringToSign = "timestamp="+timeStamp+"&body="+ params.toString();
Mac mac = Mac.getInstance("HmacSHA256");
mac.init(new SecretKeySpec(secretKey.getBytes("UTF-8"), "HmacSHA256"));
byte[] signatureBytes = mac.doFinal(stringToSign.getBytes("UTF-8"));
signature = new String(Base64.encodeBase64(signatureBytes));
return signature;
} catch (Exception e) {
e.printStackTrace();
throw new Exception("计算签名错误");
}
}
public static JSONObject post(String url, JSONObject json) throws Exception {
return (JSONObject) post(url, json, new JSONObject());
}
private static Object post(String url, JSONObject json, Object t) throws Exception {
HttpClient client = new DefaultHttpClient();
HttpPost post = new HttpPost(url);
post.setHeader("Content-Type", "application/json");
String result = "";
try {
StringEntity s = new StringEntity(json.toJSONString(), "utf-8");
post.setEntity(s);
// 发送请求
HttpResponse httpResponse = client.execute(post);
// 获取响应输入流
InputStream inStream = httpResponse.getEntity().getContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(inStream, "utf-8"));
StringBuilder strber = new StringBuilder();
String line = null;
while ((line = reader.readLine()) != null){
strber.append(line + "\n");
}
inStream.close();
result = strber.toString();
if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
return JSONObject.parseObject(result, t.getClass());
} else {
return null;
}
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("请求异常");
}
}
/**
* 获取项目标签
*
* @return
*/
public static List<ProjectLabel> listProjectLabel(String projectId) {
List<ProjectLabel> labels = null;
try {
JSONObject body = new JSONObject();
body.put("id", projectId);
long timeStamp = new Date().getTime();
String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey);
JSONObject param = new JSONObject();
param.put("signature", signature);
param.put("timeStamp", timeStamp);
param.put("corpid", corpId);
param.put("body", body);
JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/project/getProjectLabel", param);
if ((Integer)result.get("code") == 0) {
JSONArray jsonArray = result.getJSONArray("data");
labels = jsonArray.toJavaList(ProjectLabel.class);
}
} catch (Exception e) {
e.printStackTrace();
}
return labels;
}
/**
* 获取项目列表
* @return
*/
public static List<Project> listProject() {
List<Project> project = null;
try {
JSONObject body = new JSONObject();
body.put("limitStart", "0");
body.put("limitNum", "50");
long timeStamp = new Date().getTime();
String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey);
JSONObject param = new JSONObject();
param.put("signature", signature);
param.put("timeStamp", timeStamp);
param.put("corpid", corpId);
param.put("body", body);
JSONObject result = ApiUtil.post("http://api.antbim.net/ant_api/project/getProjectList", param);
if ((Integer)result.get("code") == 0) {
if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) {
JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list");
project = jsonArray.toJavaList(Project.class);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return project;
}
/**
* 获取项目下任务清单列表
*
* @return
*/
public static JSONObject listInventory(String projectId) {
JSONObject result = new JSONObject();
try {
JSONObject body = new JSONObject();
body.put("limitStart", "0");
body.put("limitNum", "50");
body.put("projectId", projectId);
long timeStamp = new Date().getTime();
String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey);
JSONObject param = new JSONObject();
param.put("signature", signature);
param.put("timeStamp", timeStamp);
param.put("corpid", corpId);
param.put("body", body);
result = ApiUtil.post("https://api.antbim.net/ant_api/project/inventory/getInventoryList", param);
} catch (Exception e) {
e.printStackTrace();
}
return result;
}
/**
* 获取项目下任务类型列表
* @return
*/
public static JSONObject listTaskType(String projectId) {
JSONObject result = new JSONObject();
try {
JSONObject body = new JSONObject();
body.put("limitStart", "0");
body.put("limitNum", "50");
body.put("projectId", projectId);
long timeStamp = new Date().getTime();
String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey);
JSONObject param = new JSONObject();
param.put("signature", signature);
param.put("timeStamp", timeStamp);
param.put("corpid", corpId);
param.put("body", body);
result = ApiUtil.post("https://api.antbim.net/ant_api/project/tasktype/getTasktypeList", param);
} catch (Exception e) {
e.printStackTrace();
}
return result;
}
/**
* 获取项目内任务列表
*
* @return
*/
public static List<Task> listTaskByProject(String projectId, String startTime, String endTime, int limitStart, int limitNum) {
List<Task> tasks = null;
try {
JSONObject body = new JSONObject();
body.put("limitStart", limitStart);
body.put("limitNum", limitNum);
body.put("projectId", projectId);
body.put("createtimeStart", startTime);
body.put("createtimeEnd", endTime);
long timeStamp = new Date().getTime();
String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey);
JSONObject param = new JSONObject();
param.put("signature", signature);
param.put("timeStamp", timeStamp);
param.put("corpid", corpId);
param.put("body", body);
JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByProject", param);
if ((Integer)result.get("code") == 0) {
if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) {
JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list");
tasks = jsonArray.toJavaList(Task.class);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return tasks;
}
/**
* 获取人员任务列表
*/
public static List<Task> listTaskByPerson(String userid, String startTime, String endTime, int limitStart, int limitNum) {
List<Task> tasks = null;
try {
JSONObject body = new JSONObject();
body.put("limitStart", limitStart);
body.put("limitNum", limitNum);
body.put("userid", userid);
body.put("type", "");
body.put("timezone", "");
body.put("createtimeStart", startTime);
body.put("createtimeEnd", endTime);
long timeStamp = new Date().getTime();
String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey);
JSONObject param = new JSONObject();
param.put("signature", signature);
param.put("timeStamp", timeStamp);
param.put("corpid", corpId);
param.put("body", body);
JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByPerson", param);
int code = (Integer)result.get("code");
if (code == 0) {
if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) {
JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list");
tasks = jsonArray.toJavaList(Task.class);
}
} else if (code == 90001){
throw new RuntimeException("每个企业调用单个接口的频率不可超过60次/分钟,会返回错误码90001:listTaskByPerson");
} else if (code == 90009) {
throw new RuntimeException("每个企业单个接口每天的最大上限为1000次,若超出会返回错误码90009:listTaskByPerson");
}
} catch (Exception ex) {
ex.printStackTrace();
}
return tasks;
}
/**
* 获取人员任务列表
*/
public static List<Task> listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) {
return listTaskByPerson(null, startTime, endTime, limitStart, limitNum);
}
/**
* 获取任务详情
*
* @return
*/
public static TaskDetail getTaskDetail(String taskId) {
TaskDetail detail = null;
try {
JSONObject body = new JSONObject();
body.put("id", taskId);
long timeStamp = new Date().getTime();
String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey);
JSONObject param = new JSONObject();
param.put("signature", signature);
param.put("timeStamp", timeStamp);
param.put("corpid", corpId);
param.put("body", body);
JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskDetail", param);
int code = (Integer)result.get("code");
if (code == 0) {
JSONObject obj = JSONObject.parseObject(String.valueOf(result.get("data")));
detail = obj.toJavaObject(TaskDetail.class);
} else if (code == 90001){
Thread.sleep(1000 * 60);
return getTaskDetail(taskId);
//throw new RuntimeException("每个企业调用单个接口的频率不可超过60次/分钟,会返回错误码90001:getTaskDetail");
} else if (code == 90009) {
throw new RuntimeException("每个企业单个接口每天的最大上限为1000次,若超出会返回错误码90009:getTaskDetail");
}
} catch (Exception e) {
//e.printStackTrace();
}
return detail;
}
public static void main(String[] args) {
//List<Project> projects = ApiUtil.listProject();
//System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50));
//ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e");
System.out.println(ApiUtil.getTaskDetail("be85282dc57d46e3a36fc0801107e605"));
//System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21"));
//System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59"));
// int limitStart = 0;
// List<Task> tasks = ApiUtil.listTaskByPerson("055551305431136413","2021-03-01 00:00:00", "2021-04-30 23:59:59", limitStart, 50);
// List<Task> tempTasks = tasks;
//
// while (tasks != null && tempTasks != null && tempTasks.size() == 50) {
// limitStart += 50;
// tempTasks = ApiUtil.listTaskByPerson("055551305431136413","2021-03-01 00:00:00", "2021-04-30 23:59:59", limitStart, 50);
// if (CollectionUtils.isNotEmpty(tempTasks)) {
// tasks.addAll(tempTasks);
// }
// }
// for (Task task : tasks) {
// if ("穆良书".equals(task.getResponse().getName())) {
// System.out.println(task);
// }
// }
}
}
@@ -0,0 +1,92 @@
package com.fangjia.script.statistics;
/**
* @author shenjian
* @since 2021/5/8
*/
public class Project {
private Integer taskCount;
private Integer taskendCount;
private String name;
private CreateUser createUser;
private String id;
private String state;
private String createDate;
public Integer getTaskCount() {
return taskCount;
}
public void setTaskCount(Integer taskCount) {
this.taskCount = taskCount;
}
public Integer getTaskendCount() {
return taskendCount;
}
public void setTaskendCount(Integer taskendCount) {
this.taskendCount = taskendCount;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public CreateUser getCreateUser() {
return createUser;
}
public void setCreateUser(CreateUser createUser) {
this.createUser = createUser;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getState() {
return state;
}
public void setState(String state) {
this.state = state;
}
public String getCreateDate() {
return createDate;
}
public void setCreateDate(String createDate) {
this.createDate = createDate;
}
}
class CreateUser {
private String name;
private String userid;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getUserid() {
return userid;
}
public void setUserid(String userid) {
this.userid = userid;
}
}
@@ -0,0 +1,57 @@
package com.fangjia.script.statistics;
/**
* @author shenjian
* @since 2021/5/8
*/
public class ProjectLabel {
private String color;
private String name;
private Groups groups;
private String id;
public String getColor() {
return color;
}
public void setColor(String color) {
this.color = color;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Groups getGroups() {
return groups;
}
public void setGroups(Groups groups) {
this.groups = groups;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}
class Groups {
private String name;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}

Some files were not shown because too many files have changed in this diff Show More