Merge remote-tracking branch 'origin/master'
This commit is contained in:
+16
-12
@@ -29,7 +29,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
|
|||||||
@Override
|
@Override
|
||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
List<String> cities = [];
|
List<String> cities = [];
|
||||||
def allCity = CommonUtils.findAllCity()
|
def allCity = ["广州","深圳","临沂"]//CommonUtils.findAllCity()
|
||||||
|
|
||||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||||
cities.addAll(allCity)
|
cities.addAll(allCity)
|
||||||
@@ -38,7 +38,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
|
|||||||
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList()
|
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList()
|
||||||
println(city)
|
println(city)
|
||||||
}
|
}
|
||||||
ExecutorService executorService = Executors.newFixedThreadPool(20)
|
ExecutorService executorService = Executors.newFixedThreadPool(1)
|
||||||
districts.each {BasicDBObject district->
|
districts.each {BasicDBObject district->
|
||||||
executorService.submit(new Calculator(district))
|
executorService.submit(new Calculator(district))
|
||||||
}
|
}
|
||||||
@@ -65,22 +65,26 @@ class Calculator extends Thread{
|
|||||||
}
|
}
|
||||||
def statMap = [:]
|
def statMap = [:]
|
||||||
|
|
||||||
DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo->
|
DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type).append("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01","yyyy-MM-dd")))).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo->
|
||||||
Date weekend = dbo.weekend
|
Date weekend = dbo.weekend
|
||||||
if(DateUtil.isLastSundayOfMonth(weekend)){
|
if(DateUtil.isLastSundayOfMonth(weekend)){
|
||||||
def trend = new BasicDBObject("city", district.city)
|
def trend = new BasicDBObject("city", district.city)
|
||||||
trend.append("region", district.region)
|
trend.append("region", district.region)
|
||||||
trend.append("district_name", district.name)
|
trend.append("district_name", district.name)
|
||||||
trend.append("estate_type", estate_type)
|
// trend.append("estate_type", estate_type)
|
||||||
trend.append("date", weekend.format("yyyyMM").toInteger())
|
trend.append("date", weekend.format("yyyyMM").toInteger())
|
||||||
|
|
||||||
|
|
||||||
def udata = new BasicDBObject();
|
def udata = new BasicDBObject();
|
||||||
udata.put("avg_price", dbo.avg_price)
|
if(dbo.avg_price_2b){
|
||||||
udata.put("credit", dbo.credit)
|
// udata.put("avg_price", dbo.avg_price)
|
||||||
udata.put("m_date", new Date())
|
udata.put("avg_price", dbo.avg_price_2b)
|
||||||
DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
|
udata.put("credit", dbo.credit)
|
||||||
statMap.put(weekend.format("yyyyMM").toInteger(), udata)
|
udata.put("m_date", new Date())
|
||||||
|
DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
|
||||||
|
statMap.put(weekend.format("yyyyMM").toInteger(), udata)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
DISTRICT_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).sort(new BasicDBObject("date", 1)).each {BasicDBObject trend->
|
DISTRICT_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).sort(new BasicDBObject("date", 1)).each {BasicDBObject trend->
|
||||||
@@ -88,14 +92,14 @@ class Calculator extends Thread{
|
|||||||
def monDate = getMomDate(date)
|
def monDate = getMomDate(date)
|
||||||
def yoyDate = getYoyDate(date)
|
def yoyDate = getYoyDate(date)
|
||||||
BasicDBObject monDbo = statMap.get(monDate)
|
BasicDBObject monDbo = statMap.get(monDate)
|
||||||
if(monDbo != null && monDbo.avg_price != 0) {
|
if(monDbo != null && monDbo.avg_price_2b != 0) {
|
||||||
trend.mom = DataUtil.roundDouble(((trend.avg_price - monDbo.avg_price)/(double)monDbo.avg_price),4)
|
trend.mom = DataUtil.roundDouble(((trend.avg_price_2b - monDbo.avg_price_2b)/(double)monDbo.avg_price_2b),4)
|
||||||
}else{
|
}else{
|
||||||
trend.mom = 0d
|
trend.mom = 0d
|
||||||
}
|
}
|
||||||
BasicDBObject yoyDbo = statMap.get(yoyDate)
|
BasicDBObject yoyDbo = statMap.get(yoyDate)
|
||||||
if(yoyDbo != null && yoyDbo.avg_price != 0){
|
if(yoyDbo != null && yoyDbo.avg_price != 0){
|
||||||
trend.yoy = DataUtil.roundDouble(((trend.avg_price - yoyDbo.avg_price)/(double)yoyDbo.avg_price),4)
|
trend.yoy = DataUtil.roundDouble(((trend.avg_price_2b - yoyDbo.avg_price_2b)/(double)yoyDbo.avg_price_2b),4)
|
||||||
}else{
|
}else{
|
||||||
trend.yoy = 0d
|
trend.yoy = 0d
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
|||||||
* 深圳公众号成交数据录入
|
* 深圳公众号成交数据录入
|
||||||
*/
|
*/
|
||||||
|
|
||||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
|
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年5月深圳成交明细.xlsx", 0, 1)
|
||||||
|
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
|||||||
+26
@@ -855,6 +855,32 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static def findAllDistrictsKey(def city, def cat, def fields){
|
||||||
|
|
||||||
|
DBObject q = new BasicDBObject();
|
||||||
|
q.put("status",0)
|
||||||
|
// q.put("status",new BasicDBObject('$ne', 0))
|
||||||
|
q.put("cat", cat);
|
||||||
|
q.put("city", city)
|
||||||
|
// q.put("loop_line_info",new BasicDBObject('$exists',false))
|
||||||
|
// q.put("name","经纬天地园二期")
|
||||||
|
|
||||||
|
DBObject keys = new BasicDBObject();
|
||||||
|
if(fields){
|
||||||
|
for(def field : fields){
|
||||||
|
keys.put(field, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||||
|
def keyMap = [:]
|
||||||
|
for(def d : dbsour){
|
||||||
|
keyMap.put(d.region+"##"+d.name, d)
|
||||||
|
}
|
||||||
|
return keyMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// public static def findAllDistrictByCity(def city, def cat, def fields){
|
// public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||||
//
|
//
|
||||||
// DBObject q = new BasicDBObject();
|
// DBObject q = new BasicDBObject();
|
||||||
|
|||||||
+9
-3
@@ -73,6 +73,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
|
|
||||||
|
|
||||||
def appendResult(def headers, def ret){
|
def appendResult(def headers, def ret){
|
||||||
|
|
||||||
def sb = new StringBuffer();
|
def sb = new StringBuffer();
|
||||||
// sb.append(headers.join("@|@"))
|
// sb.append(headers.join("@|@"))
|
||||||
// sb.append(System.getProperty("line.separator"));
|
// sb.append(System.getProperty("line.separator"));
|
||||||
@@ -121,11 +122,16 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
def day = new Date().getDate()
|
def day = new Date().getDate()
|
||||||
def startMDate = DateUtils.truncateDay(new Date()-1)
|
def startMDate = DateUtils.truncateDay(new Date()-1)
|
||||||
def endMDate = DateUtils.truncateDay(new Date())
|
def endMDate = DateUtils.truncateDay(new Date())
|
||||||
if(day == 15){
|
if(CommonDatautils.isWindow()){
|
||||||
districts = getFzDistricts(city, null, null)
|
districts = getFzDistricts(city, null, null)
|
||||||
}else{
|
} else{
|
||||||
districts = getFzDistricts(city, startMDate, endMDate)
|
if(day == 15){
|
||||||
|
districts = getFzDistricts(city, null, null)
|
||||||
|
}else{
|
||||||
|
districts = getFzDistricts(city, startMDate, endMDate)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def curPriceMap = findFzDistrictSellStat("福州",curDate)
|
def curPriceMap = findFzDistrictSellStat("福州",curDate)
|
||||||
def result = []
|
def result = []
|
||||||
for(def rMap : districts){
|
for(def rMap : districts){
|
||||||
|
|||||||
+2
-1
@@ -1,5 +1,6 @@
|
|||||||
package com.fangjia.script.fuzhou
|
package com.fangjia.script.fuzhou
|
||||||
|
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
import com.fangjia.core.util.DataUtil
|
import com.fangjia.core.util.DataUtil
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.BatchOwner
|
import com.fangjia.script.BatchOwner
|
||||||
@@ -63,7 +64,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def calcMonthlyPrice(){
|
def calcMonthlyPrice(){
|
||||||
//只有福州使用tob价格
|
//只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外
|
||||||
def citys = CommonUtils.findAllCity()
|
def citys = CommonUtils.findAllCity()
|
||||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
|
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
|
||||||
|
|
||||||
|
|||||||
+7
-10
@@ -29,11 +29,11 @@ class FzDataResultStat {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new FzDataResultStat()
|
def task = new FzDataResultStat()
|
||||||
task.priceMarkupRun();
|
// task.priceMarkupRun();
|
||||||
// task.regionPriceTrendRun()
|
// task.regionPriceTrendRun()
|
||||||
//// task.margeExcel()
|
//// task.margeExcel()
|
||||||
// task.saveCsvData()
|
// task.saveCsvData()
|
||||||
// task.testNewPrice()
|
task.testNewPrice()
|
||||||
}
|
}
|
||||||
|
|
||||||
def margeExcel(){
|
def margeExcel(){
|
||||||
@@ -82,7 +82,7 @@ class FzDataResultStat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||||
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
|
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -4, "yyyyMM")))
|
||||||
def map2 = [:]
|
def map2 = [:]
|
||||||
for(def obj : list2) {
|
for(def obj : list2) {
|
||||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||||
@@ -119,7 +119,7 @@ class FzDataResultStat {
|
|||||||
index++
|
index++
|
||||||
println row
|
println row
|
||||||
}
|
}
|
||||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
ExcelUtils.write("D:\\数据\\福州\\价格比对5_${curMonth}.xlsx", rows, "数据")
|
||||||
|
|
||||||
|
|
||||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||||
@@ -301,16 +301,13 @@ class FzDataResultStat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def testNewPrice(){
|
def testNewPrice(){
|
||||||
ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d->
|
ExcelUtils.modify("D:\\数据\\福州\\福州200个小区抽样价格核验202507.xlsx",{d->
|
||||||
if(d['A']){
|
if(d['A']){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("city", d['A'])
|
q.put("comm_id", d['A'])
|
||||||
q.put("region",d['B'])
|
|
||||||
q.put("name",d['C'])
|
|
||||||
q.put("month", 202402)
|
|
||||||
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
|
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
|
||||||
if(obj){
|
if(obj){
|
||||||
d['S'] = obj.avg_price
|
d['D'] = obj.avg_price
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
+109
-1
@@ -13,11 +13,17 @@ import com.mongodb.DBCursor
|
|||||||
import com.mongodb.DBObject
|
import com.mongodb.DBObject
|
||||||
import org.apache.solr.common.util.DateUtil
|
import org.apache.solr.common.util.DateUtil
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
class AddAvgpriceBatch {
|
class AddAvgpriceBatch {
|
||||||
|
|
||||||
private static def citys = null;
|
private static def citys = null;
|
||||||
private static def districtMap = [:];
|
private static def districtMap = [:];
|
||||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
|
||||||
|
|
||||||
static {
|
static {
|
||||||
citys = ["宜昌"]//CommonDatautils.findAllCity()
|
citys = ["宜昌"]//CommonDatautils.findAllCity()
|
||||||
@@ -26,10 +32,112 @@ class AddAvgpriceBatch {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new AddAvgpriceBatch();
|
def task = new AddAvgpriceBatch();
|
||||||
task.run1();
|
// task.run1();
|
||||||
// task.test()
|
// task.test()
|
||||||
// task.sendPriceCommand()
|
// task.sendPriceCommand()
|
||||||
// task.sendNewDistPrice()
|
// task.sendNewDistPrice()
|
||||||
|
|
||||||
|
// task.delNoExistsDistPrice()
|
||||||
|
task.checkToBprice()
|
||||||
|
}
|
||||||
|
|
||||||
|
def checkToBprice(){
|
||||||
|
def citys = CommonDatautils.findAllCity().reverse()
|
||||||
|
for(def i=0;i<citys.size();i++){ //76
|
||||||
|
def c = citys[i]
|
||||||
|
def keyDistMap = CommonDatautils.findAllDistrictsKey(c, "district", ["city","region","name"])
|
||||||
|
def rets = findNoPrices(c, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), keyDistMap)
|
||||||
|
println c + " "+rets.size()
|
||||||
|
addCommand(rets)
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def addCommand(rets){
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||||
|
|
||||||
|
for(def i=0;i<rets.size();i++){
|
||||||
|
def m = rets[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
CommonUtils.recalculationAverage(m.city, m.district, m.name, m.estate_type, m.weekend,false,0)
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
})
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def findNoPrices(c, weekend, keyDistMap){
|
||||||
|
def q = new BasicDBObject("city", c).append("weekend", new BasicDBObject('$gte', weekend)).append("avg_price_2b", null)
|
||||||
|
|
||||||
|
def keys = new BasicDBObject()
|
||||||
|
keys.put("city", 1)
|
||||||
|
keys.put("region", 1)
|
||||||
|
keys.put("district_name", 1)
|
||||||
|
keys.put("estate_type", 1)
|
||||||
|
keys.put("avg_price_2b", 1)
|
||||||
|
keys.put("weekend", 1)
|
||||||
|
def dbcursor = DIST_STAT_COLL.find(q, keys).batchSize(5000)
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def distObj = keyDistMap.get(d.region+"##"+d.district_name)
|
||||||
|
if(distObj){
|
||||||
|
if(!d.avg_price_2b) {
|
||||||
|
rets << ["city": d.city, "district": d.region, "name": d.district_name,"estate_type":d.estate_type, "weekend": d.weekend]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
return rets
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def delNoExistsDistPrice(){
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
def index = 0
|
||||||
|
for(def c : citys){
|
||||||
|
index ++
|
||||||
|
if(index<250){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
println index +" "+c
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name"])
|
||||||
|
def nameKeys = []
|
||||||
|
for(def dist : dists){
|
||||||
|
nameKeys << dist.region+"#"+dist.name
|
||||||
|
}
|
||||||
|
delDistPrice(c, nameKeys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def delDistPrice(city, nameKeys){
|
||||||
|
def q = new BasicDBObject("city", city)
|
||||||
|
|
||||||
|
def keys = new BasicDBObject()
|
||||||
|
keys.put("city", 1)
|
||||||
|
keys.put("region", 1)
|
||||||
|
keys.put("district_name", 1)
|
||||||
|
def dbcursor = DIST_STAT_COLL.find(q, keys).batchSize(5000)
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def key = d.region +"#"+d.district_name
|
||||||
|
if(!nameKeys.contains(key)){
|
||||||
|
DIST_STAT_COLL.remove(new BasicDBObject("_id", d._id))
|
||||||
|
println "删除:"+city +" "+ d.district_name
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def sendNewDistPrice(){
|
def sendNewDistPrice(){
|
||||||
|
|||||||
@@ -1,21 +1,225 @@
|
|||||||
package com.fangjia.script.gj
|
package com.fangjia.script.gj
|
||||||
|
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.gj.ccb.FileUtils
|
||||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
import com.fangjia.script.util.CommonUtils
|
import com.fangjia.script.util.CommonUtils
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import org.apache.poi.xwpf.usermodel.XWPFDocument
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.Semaphore
|
import java.util.concurrent.Semaphore
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
class DistrictStat {
|
class DistrictStat {
|
||||||
|
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
|
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||||
|
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new DistrictStat();
|
def task = new DistrictStat();
|
||||||
|
|
||||||
// task.run()
|
// task.run()
|
||||||
task.run("无锡")
|
// task.run("无锡")
|
||||||
|
|
||||||
|
// 合肥市
|
||||||
|
// 芜湖市
|
||||||
|
// 安庆市
|
||||||
|
// 马鞍山市
|
||||||
|
// 宣称市
|
||||||
|
// 六安市
|
||||||
|
// def citys = ["宣城","合肥","芜湖","安庆","马鞍山","六安"]
|
||||||
|
// for(def city : citys){
|
||||||
|
// task.stat(city)
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
// task.cityStat()
|
||||||
|
task.cityStat1()
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def cityStat1(){
|
||||||
|
def index = 0
|
||||||
|
def rets = []
|
||||||
|
ExcelUtils.modify("D:\\数据\\小区\\城市数量统计1.xlsx",{d->
|
||||||
|
|
||||||
|
def districts = null
|
||||||
|
if(d['D']=="义乌"){
|
||||||
|
districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"])
|
||||||
|
def newDists = []
|
||||||
|
for(def dist : districts){
|
||||||
|
if(dist.region =="义乌"){
|
||||||
|
newDists << dist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
districts = newDists
|
||||||
|
}else if(d['D']=="金华"){
|
||||||
|
districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"])
|
||||||
|
def newDists = []
|
||||||
|
for(def dist : districts){
|
||||||
|
if(dist.region !="义乌"){
|
||||||
|
newDists << dist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
districts = newDists
|
||||||
|
}else if(d['D']=="无锡"){
|
||||||
|
districts = CommonDatautils.findAllDistrictByCity("无锡", "district", ["region","name"])
|
||||||
|
def newDists = []
|
||||||
|
for(def dist : districts){
|
||||||
|
if(dist.region !="江阴"){
|
||||||
|
newDists << dist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
districts = newDists
|
||||||
|
}else if(d['D']=="苏州"){
|
||||||
|
districts = CommonDatautils.findAllDistrictByCity("苏州", "district", ["region","name"])
|
||||||
|
def newDists = []
|
||||||
|
for(def dist : districts){
|
||||||
|
if(dist.region !="常熟"){
|
||||||
|
newDists << dist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
districts = newDists
|
||||||
|
}else if(d['D']=="济南"){
|
||||||
|
districts = CommonDatautils.findAllDistrictByCity("济南", "district", ["region","name"])
|
||||||
|
def newDists = []
|
||||||
|
for(def dist : districts){
|
||||||
|
if(dist.region !="莱芜"){
|
||||||
|
newDists << dist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
districts = newDists
|
||||||
|
}else{
|
||||||
|
districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["region","name"])
|
||||||
|
}
|
||||||
|
|
||||||
|
d['F'] = districts.size()
|
||||||
|
if(districts){
|
||||||
|
d['E'] = "是"
|
||||||
|
}else{
|
||||||
|
d['E'] = "否"
|
||||||
|
}
|
||||||
|
|
||||||
|
def prov = null
|
||||||
|
if(districts){
|
||||||
|
index ++
|
||||||
|
def data = fullCityMap.get(d['D'])
|
||||||
|
if(data){
|
||||||
|
prov = data.province
|
||||||
|
}
|
||||||
|
def str = districts.collect { it.name }.join(", ")
|
||||||
|
rets << ["id":index, "type":"A", "province":prov, "city":d['D'], "names":str]
|
||||||
|
|
||||||
|
createDoc(rets)
|
||||||
|
}
|
||||||
|
}, 0)
|
||||||
|
|
||||||
|
// FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表1.dat", [], "utf-8", false);
|
||||||
|
}
|
||||||
|
|
||||||
|
def createDoc(dataList){
|
||||||
|
// 创建 Word 文档
|
||||||
|
def doc = new XWPFDocument()
|
||||||
|
def table = doc.createTable(dataList.size() + 1, 5) // 行数=数据行+表头,列数=3
|
||||||
|
|
||||||
|
// 设置表头
|
||||||
|
def headerRow = table.getRow(0)
|
||||||
|
|
||||||
|
headerRow.getCell(0).setText("序号")
|
||||||
|
headerRow.getCell(1).setText("类型")
|
||||||
|
headerRow.getCell(2).setText("省份")
|
||||||
|
headerRow.getCell(3).setText("城市")
|
||||||
|
headerRow.getCell(4).setText("楼盘名称")
|
||||||
|
|
||||||
|
// 填充数据
|
||||||
|
dataList.eachWithIndex { item, index ->
|
||||||
|
def row = table.getRow(index+1) // 跳过表头
|
||||||
|
row.getCell(0).setText(item.id.toString())
|
||||||
|
row.getCell(1).setText(item.type.toString())
|
||||||
|
row.getCell(2).setText(item.province.toString())
|
||||||
|
row.getCell(3).setText(item.city.toString())
|
||||||
|
row.getCell(4).setText(item.names.toString())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 保存到文件
|
||||||
|
def outputFile = new File("A类城市楼盘列表.docx")
|
||||||
|
doc.write(new FileOutputStream(outputFile))
|
||||||
|
println "Word 表格生成成功: ${outputFile.absolutePath}"
|
||||||
|
}
|
||||||
|
|
||||||
|
def cityStat(){
|
||||||
|
|
||||||
|
def index = 0
|
||||||
|
def rets = []
|
||||||
|
ExcelUtils.modify("D:\\数据\\小区\\城市数量统计.xlsx",{d->
|
||||||
|
def districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["city", "region", "name"])
|
||||||
|
if(districts){
|
||||||
|
d['F'] = districts.size()
|
||||||
|
d['E'] = "是"
|
||||||
|
|
||||||
|
rets << ["序号","类型","省份","城市","楼盘名称","金华"]
|
||||||
|
def data = fullCityMap.get(d['D'])
|
||||||
|
def prov = null
|
||||||
|
if(data){
|
||||||
|
prov = data.province
|
||||||
|
}
|
||||||
|
|
||||||
|
for(def dist : districts){
|
||||||
|
index++
|
||||||
|
rets << [index, "A", prov,dist.city, dist.name, dist.region]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 0)
|
||||||
|
|
||||||
|
// ExcelUtils.write("D:\\数据\\小区\\A类城市楼盘列表.xlsx", rets, "楼盘数据详情列表")
|
||||||
|
FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表.dat", [], "utf-8", false);
|
||||||
|
}
|
||||||
|
|
||||||
|
def stat(city){
|
||||||
|
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name","alias"])
|
||||||
|
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||||
|
|
||||||
|
def result = Collections.synchronizedList(new ArrayList());
|
||||||
|
def countResult = Collections.synchronizedList(new ArrayList());
|
||||||
|
def rets = []
|
||||||
|
// def houseTotal = CommonDatautils.findHouseCountByDate(city, null, null, null, "sell", 20240611)
|
||||||
|
rets << city
|
||||||
|
rets << districts.size()
|
||||||
|
// rets << houseTotal
|
||||||
|
for(def i=0;i<districts.size();i++){
|
||||||
|
final def o = districts[i];
|
||||||
|
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
def count = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20240611)
|
||||||
|
if(count){
|
||||||
|
result << o
|
||||||
|
countResult << count
|
||||||
|
}
|
||||||
|
println totalCount.get() +" "+o.region +" "+o.name+" "+count
|
||||||
|
} catch (Exception e) {
|
||||||
|
println totalCount.get() +" 查房源Error: "+city+" "+ o.region+" "+o.name+" "+e.getMessage()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
})
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
rets << countResult.sum()
|
||||||
|
rets << result.size()
|
||||||
|
|
||||||
|
println rets
|
||||||
}
|
}
|
||||||
def run(def city){
|
def run(def city){
|
||||||
def weekend = DateUtils.parseFromTightYYYYMMDD("2020-10-25")
|
def weekend = DateUtils.parseFromTightYYYYMMDD("2020-10-25")
|
||||||
|
|||||||
+4
-2
@@ -64,7 +64,9 @@ class FangjiatongUserStat {
|
|||||||
|
|
||||||
def stat(){
|
def stat(){
|
||||||
def users = findUses(12)
|
def users = findUses(12)
|
||||||
def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
|
// def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
|
||||||
|
// def months = ["2020-01-01","2020-02-01","2020-03-01","2020-04-01","2020-05-01","2020-06-01","2020-07-01","2020-08-01","2020-09-01","2020-10-01","2020-11-01","2020-12-01","2021-01-01"]
|
||||||
|
def months = ["2022-01-01","2022-02-01","2022-03-01","2022-04-01","2022-05-01","2022-06-01","2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01"]
|
||||||
def rets = []
|
def rets = []
|
||||||
def header = ["用户id","支行名称", "备注名称"]
|
def header = ["用户id","支行名称", "备注名称"]
|
||||||
months.each{m->
|
months.each{m->
|
||||||
@@ -87,7 +89,7 @@ class FangjiatongUserStat {
|
|||||||
}
|
}
|
||||||
rets << ret
|
rets << ret
|
||||||
}
|
}
|
||||||
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019")
|
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2022")
|
||||||
}
|
}
|
||||||
|
|
||||||
def findUses(role){
|
def findUses(role){
|
||||||
|
|||||||
+20
-1
@@ -27,10 +27,29 @@ class SuzouPocTest {
|
|||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new SuzouPocTest()
|
def task = new SuzouPocTest()
|
||||||
// task.run()
|
// task.run()
|
||||||
task.newRun()
|
// task.newRun()
|
||||||
|
task.test()
|
||||||
|
}
|
||||||
|
|
||||||
|
def test(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\上海联合产权交易所\\第1批拟处置房产清单(长城大厦、台山碧桂园).xlsx", { d ->
|
||||||
|
if(d['O']){
|
||||||
|
def floorAdds = d['I'].split("/")
|
||||||
|
def ret1 = OpenApiClient.addressEvaluation(d['O'], null, d['D'], d['D'], d['K'], floorAdds[0], floorAdds[1])
|
||||||
|
if (ret1) {
|
||||||
|
d['N'] = ret1.totalPrice
|
||||||
|
d['M'] = ret1.avgPrice
|
||||||
|
d['P'] = ret1.district
|
||||||
|
d['Q'] = ret1.name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def newRun(){
|
def newRun(){
|
||||||
|
|
||||||
println("start:"+new Date().toString())
|
println("start:"+new Date().toString())
|
||||||
def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls"
|
def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls"
|
||||||
newMatch(filePath)
|
newMatch(filePath)
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ class FileUtils {
|
|||||||
def sb = new StringBuffer();
|
def sb = new StringBuffer();
|
||||||
|
|
||||||
if(headers){
|
if(headers){
|
||||||
sb.append(headers.join("|@|"))
|
sb.append(headers.join("@"))
|
||||||
sb.append("\n");
|
sb.append("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -32,7 +32,7 @@ class FileUtils {
|
|||||||
for(def row : ret){
|
for(def row : ret){
|
||||||
if(row){
|
if(row){
|
||||||
index++
|
index++
|
||||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
sb.append(StringUtils.join(row.toArray(), "@"))
|
||||||
sb.append("\n");
|
sb.append("\n");
|
||||||
|
|
||||||
if(index%10000==0){
|
if(index%10000==0){
|
||||||
|
|||||||
+583
@@ -0,0 +1,583 @@
|
|||||||
|
package com.fangjia.script.gj.crawl
|
||||||
|
|
||||||
|
import com.fangjia.core.util.HttpRequest
|
||||||
|
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||||
|
import com.fangjia.script.newDistrict.InvokeHttpRequest
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import net.sf.json.JSONObject
|
||||||
|
import net.sf.json.groovy.JsonSlurper
|
||||||
|
import org.jsoup.Jsoup
|
||||||
|
import org.jsoup.nodes.Element
|
||||||
|
import org.jsoup.select.Elements
|
||||||
|
|
||||||
|
class SzGovHouseCrawl {
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new SzGovHouseCrawl();
|
||||||
|
task.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
def index_content = """
|
||||||
|
|
||||||
|
|
||||||
|
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<title>一手房源公示</title>
|
||||||
|
<link href="css/marster.css" rel="stylesheet" type="text/css" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<form name="Form1" method="post" action="index.aspx" id="Form1">
|
||||||
|
<div>
|
||||||
|
<input type="hidden" name="__EVENTTARGET" id="__EVENTTARGET" value="" />
|
||||||
|
<input type="hidden" name="__EVENTARGUMENT" id="__EVENTARGUMENT" value="" />
|
||||||
|
<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="nFQUad+408Aq5zCXFL9mwJbwfbUjHqOTwTmTIzIV/nn2wuuQIXExg/fWXnK8Ll6o8tHsGFJtGbKc87jIO8eGWmN56r/iP2F/Oi3sPuUM14rN/OFTR/y1yS4eQXiTUniZCpvc+hwWUXM6La+cm63hbAskQFo2Geb/8JeX/kCREa/vJqSoFTxJFuxoDevNRV5QeytYQ8PxV89z/L/KAGHwo7sR66wzuWciXwvLcXa6BWaA9aU953Rr1QnFrqy3QrAnmLmxfEtEbvw6cySgECeARwElfHdG0dKGni1inFrFf3lYDdNAL2gLoQoNFBIbDj/vXaYZQoAypynsqc8hiZotSa5aqehFlDxDu3TO3jRcWKQDue55UoGZURl8WyjlfLpebiAdSQirRRrLRQHcb4Baf7U0y/JiFFk0A75I9bfJM72eq16jjjhV9++DNXJRUn5KGMSn5h+PxwcsS8rSga0KRwsM9LDylCsNkedVucDhMCGrNwmbhAsnUIiMD7Va8wQizQllRtDJWPFZRU+YPUWxqRkfJnTEzdl8MQw7eEC6rweywfrDn3VLvw8RkRqAdaSyA5K3z4VXR0QVuBcX6teex3UIQQep9oWVvK6FGlpHEfkqEPYdwnyoEoGgRJHk/R0xhCx2ahrlYI04dbT2GaugM2JDVBnGWhQBYeylLukCVpBnaxUgiGG1Dx6lUBbLi0yFt3GkdYzi5XeTzQoGsMTbdVMBvcIqqFOs9dZBfebrWrpwTnAgUldkrtyw6j9Lf0sKPCUifvCCg8Ks1OnniSkpI1zJ0sfseGrnPQv7357IEg6UWyfvMFr3jjzqcX0QV35ajJMpzfaKIYGlzzIkQwEfjhNU83+nhI3yct8nyfmE2lROKDzOdCYFmlOPVQ1yF0vdPWs2cmx3SeQl53r1nPokHRQFuYKif4B7Tx/GN/1jPoKtrqBekL/8+ewdF1YAL8z7okg4LQ7DWaITUPvZmP7p/kAqilgKVMzqAOmdl5FzK1RwDn1fh6Ciw79aM3MPYwzkfxwA9S9uo67RuF4DHbVkuob/SJ1JGfXvFNJEvm/45UJWBFKEYWnmJJLvbeZSSZ455ox0aNLtzsTpTm3yr7S+LVNqTQIxEdFMed1gC/9caV753lkuEAHlxWwZHE3eULOw04aM8w==" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script type="text/javascript">
|
||||||
|
//<![CDATA[
|
||||||
|
var theForm = document.forms['Form1'];
|
||||||
|
if (!theForm) {
|
||||||
|
theForm = document.Form1;
|
||||||
|
}
|
||||||
|
function __doPostBack(eventTarget, eventArgument) {
|
||||||
|
if (!theForm.onsubmit || (theForm.onsubmit() != false)) {
|
||||||
|
theForm.__EVENTTARGET.value = eventTarget;
|
||||||
|
theForm.__EVENTARGUMENT.value = eventArgument;
|
||||||
|
theForm.submit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//]]>
|
||||||
|
</script>
|
||||||
|
|
||||||
|
|
||||||
|
<script src="/ris/WebResource.axd?d=oAb-PGCUASjSAEHmsWLcW3nRSbp0kTk2wHZPA_xOoR1TX097F_c8lWeB0anDU_zTJ7nGX_3dSaDy8zzgj1ZXo1L1Sc41&t=638314433260000000" type="text/javascript"></script>
|
||||||
|
|
||||||
|
|
||||||
|
<script src="/ris/ScriptResource.axd?d=ez7z8Rk-sRUjmvQcOq01fhxlJ1QkOJwcsZyID_Ni3ynenZjRnecb_-APpFxKSgOMhcW8F7rusL02bjz8mZvC6mdZ5fFUnv7hA8P09LlmyTRz8AzNNpmFk65QF_tFzWpkOb3lJg2&t=636978664134464033" type="text/javascript"></script>
|
||||||
|
<script src="/ris/ScriptResource.axd?d=3ACyCjbvPx1mr6XvFjwUpUrueEjvCnwM6jay2gUKFWC2bAv9PJDw7_w8e-FijIouXVInB0UX5vw0I3Fbx5YA92gSlMgpLLv3eu8tE_KjYAT1gVhsDve7JfQShCjQ8wBnrY5v-Q2&t=636978664134464033" type="text/javascript"></script>
|
||||||
|
<div>
|
||||||
|
|
||||||
|
\t<input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="2A35A6B2" />
|
||||||
|
\t<input type="hidden" name="__VIEWSTATEENCRYPTED" id="__VIEWSTATEENCRYPTED" value="" />
|
||||||
|
\t<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="c/yjhDZvdy6HmhcL4EkG1J1Z3q6SMKFIycFC2eMEcd9PHzbGDxG56wrvw1REP793VkDqjnfrk1++ArpjUL0ql/yAB3MB5SUwn7h0YzAXw3/YG2ipVYdaOarImjt0o8JYuszLTaNRVxXmK8KTgUXOhpL0mroOqigxEaXUaeSKml9LBl2xzJttjM+uErnLyCLXSyLW0qjXWs+pCPqD5SgIhBUuVmg=" />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="wrap">
|
||||||
|
<div class="record fix mb10">
|
||||||
|
|
||||||
|
<script type="text/javascript">
|
||||||
|
//<![CDATA[
|
||||||
|
Sys.WebForms.PageRequestManager._initialize('scriptManager2', document.getElementById('Form1'));
|
||||||
|
Sys.WebForms.PageRequestManager.getInstance()._updateControls(['tupdatepanel2'], [], [], 90);
|
||||||
|
//]]>
|
||||||
|
</script>
|
||||||
|
|
||||||
|
|
||||||
|
<div class="recordLinkbox recordLink">
|
||||||
|
所在区:<a id="hypAll" href="javascript:__doPostBack('hypAll','')" style="color:Red;">全市</a>
|
||||||
|
<a id="hypBa" href="javascript:__doPostBack('hypBa','')" style="color:Black;">宝安</a>
|
||||||
|
<a id="hypFt" href="javascript:__doPostBack('hypFt','')" style="color:Black;">福田</a>
|
||||||
|
<a id="hypLg" href="javascript:__doPostBack('hypLg','')" style="color:Black;">龙岗</a>
|
||||||
|
<a id="hypLh" href="javascript:__doPostBack('hypLh','')" style="color:Black;">罗湖</a>
|
||||||
|
<a id="hypNs" href="javascript:__doPostBack('hypNs','')" style="color:Black;">南山</a>
|
||||||
|
<a id="hypQh" href="javascript:__doPostBack('hypQh','')" style="color:Black;">前海</a>
|
||||||
|
<a id="hypYt" href="javascript:__doPostBack('hypYt','')" style="color:Black;">盐田</a>
|
||||||
|
<a id="hypLhQ" href="javascript:__doPostBack('hypLhQ','')" style="color:Black;">龙华</a>
|
||||||
|
<a id="hypGm" href="javascript:__doPostBack('hypGm','')" style="color:Black;">光明</a>
|
||||||
|
<a id="hypPs" href="javascript:__doPostBack('hypPs','')" style="color:Black;">坪山</a>
|
||||||
|
<a id="hypDp" href="javascript:__doPostBack('hypDp','')" style="color:Black;">大鹏</a>
|
||||||
|
<a id="hypSSHZ" href="javascript:__doPostBack('hypSSHZ','')">深汕合作</a></div>
|
||||||
|
<div class="deveBox" style="line-height: 0px;">
|
||||||
|
<p class="left mr25">
|
||||||
|
项目查询 <input name="tep_name" type="text" id="tep_name" style="height:28px;width:150px;" /></p>
|
||||||
|
<p class="left mr25">
|
||||||
|
开发企业 <input name="organ_name" type="text" value="深圳市城建产业园发展有限公司" id="organ_name" style="height:28px;width:150px;" /></p>
|
||||||
|
<p class="left mr25">
|
||||||
|
宗地地址 <input name="site_address" type="text" id="site_address" style="height:28px;width:150px;" /></p>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<input type="submit" name="Button1" value="查询" id="Button1" style="background: #1956c6;
|
||||||
|
width: 75px; height: 28px; line-height: 28px; display: block; text-align: center; cursor:pointer;
|
||||||
|
font-size: 14px; color: #fff;border-radius: 2px;" />
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<div id="updatepanel2">
|
||||||
|
\t
|
||||||
|
<div class="record fix">
|
||||||
|
<div class="fix">
|
||||||
|
|
||||||
|
<table class="table ta-c bor-b-1 table-white">
|
||||||
|
<tr>
|
||||||
|
<th style="width: 92px;">
|
||||||
|
序号
|
||||||
|
</th>
|
||||||
|
<th style="width: 220px;">
|
||||||
|
预售证号
|
||||||
|
</th>
|
||||||
|
<th style="width: 312px;">
|
||||||
|
项目名称
|
||||||
|
</th>
|
||||||
|
<th style="width: 270px;">
|
||||||
|
开发企业
|
||||||
|
</th>
|
||||||
|
<th style="width: 110px;">
|
||||||
|
所在区
|
||||||
|
</th>
|
||||||
|
<th style="width: 132px;">
|
||||||
|
批准时间
|
||||||
|
</th>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
<tr class="">
|
||||||
|
<td>
|
||||||
|
1
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a href='./certdetail.aspx?id=133423' target='_self'>
|
||||||
|
深房许字(2024)罗湖002号</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a href='projectdetail.aspx?id=133423' target='_self'>
|
||||||
|
凌云华府</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
深圳市城建产业园发展有限公司
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
罗湖
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
2024-06-27
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
<tr class="bg-grayf7">
|
||||||
|
<td>
|
||||||
|
2
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a href='./certdetail.aspx?id=124226' target='_self'>
|
||||||
|
深房许字(2022)罗湖007号</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a href='projectdetail.aspx?id=124226' target='_self'>
|
||||||
|
高投大厦</a>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
深圳市城建产业园发展有限公司
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
罗湖
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
2022-12-16
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
<tr style="height:18px;">
|
||||||
|
<td colspan="6">
|
||||||
|
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</tr></table>
|
||||||
|
</div>
|
||||||
|
<div class="page">
|
||||||
|
<div class="pages pagehtml right">
|
||||||
|
|
||||||
|
\t<!-- AspNetPager V7.0.2 for VS2005 & VS2008 Copyright:2003-2007 Webdiyer (www.webdiyer.com) -->
|
||||||
|
\t<!--记录总数只有一页,AspNetPager已自动隐藏,若需在只有一页数据时显示AspNetPager,请将AlwaysShow属性值设为true!-->
|
||||||
|
\t<!-- AspNetPager V7.0.2 for VS2005 & VS2008 End -->
|
||||||
|
|
||||||
|
\t
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
<script type="text/javascript">
|
||||||
|
//<![CDATA[
|
||||||
|
Sys.Application.initialize();
|
||||||
|
//]]>
|
||||||
|
</script>
|
||||||
|
</form>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
getRoomRefDetail("https://zjj.sz.gov.cn/ris/bol/szfdc/housedetail.aspx?id=1979350",[:])
|
||||||
|
def doc = Jsoup.parse(index_content);
|
||||||
|
Elements trs = doc.select("table tr")
|
||||||
|
def name = "深圳市城建产业园发展有限公司"
|
||||||
|
def results = []
|
||||||
|
def header = ["楼层","房间号","单元号","销售状态","楼栋名称","预售许可证号","项目名称","所属项目地址","开发商名称","开盘日期","监管银行","监管账户名称","监管账户账号","房屋用途","户型","预测总建筑面积","预测套内建筑面积","预测公摊建筑面积","实际总建筑面积","实际套内建筑面积","实际公摊建筑面积","城市","拟售价格-按建筑面积计","拟售价格-按套内建筑面积计","预售总套数","现售总套数","栋数"]
|
||||||
|
results << header
|
||||||
|
//开发商查询结果页
|
||||||
|
println trs.size()-2 +" 预售证"
|
||||||
|
for(def i=1 ;i<trs.size();i++){
|
||||||
|
def href1 = trs[i].select("td:eq(1) a").attr("href")
|
||||||
|
if(href1){
|
||||||
|
href1 = "https://zjj.sz.gov.cn/ris/bol/szfdc"+ href1.substring(1)
|
||||||
|
def href2 = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+trs[i].select("td:eq(2) a").attr("href")
|
||||||
|
|
||||||
|
def sellObjMap = getSellCodeObject(href1) //预售证页面
|
||||||
|
println i+ " 预售证页面 加载完成"
|
||||||
|
def projectMap = getProjectDetail(href2) //抓取项目页面
|
||||||
|
println i+ " 预售证页面 抓取项目页面"
|
||||||
|
if(projectMap){
|
||||||
|
def projList = projectMap.projList
|
||||||
|
for(def pro : projList){
|
||||||
|
def nowsellrooms = pro.yusell_roomDetails
|
||||||
|
for(def yusell : nowsellrooms){
|
||||||
|
def row = []
|
||||||
|
row << yusell["楼层"]
|
||||||
|
row << yusell["房号"]
|
||||||
|
row << yusell["座号"]
|
||||||
|
row << yusell["房号状态"]
|
||||||
|
row << pro["楼名"]
|
||||||
|
row << sellObjMap['许可证号']
|
||||||
|
row << sellObjMap['项目名称']
|
||||||
|
row << sellObjMap['所在位置']
|
||||||
|
row << sellObjMap['发展商']
|
||||||
|
row << sellObjMap['发证日期 (开始销售日期)']
|
||||||
|
row << projectMap['价款监管机构1']
|
||||||
|
row << projectMap['账户名称']
|
||||||
|
row << projectMap['账号']
|
||||||
|
row << yusell["用途"]
|
||||||
|
row << ""
|
||||||
|
row << yusell["预售-建筑面积"]
|
||||||
|
row << yusell["预售-套内建筑面积"]
|
||||||
|
row << yusell["预售-分摊面积"]
|
||||||
|
row << yusell["竣工-建筑面积"]
|
||||||
|
row << yusell["竣工-套内建筑面积"]
|
||||||
|
row << yusell["竣工-分摊面积"]
|
||||||
|
row << "深圳"
|
||||||
|
row << yusell["拟售价格-按建筑面积计"]
|
||||||
|
row << yusell["拟售价格-按套内建筑面积计"]
|
||||||
|
row << projectMap['预售总套数']
|
||||||
|
row << projectMap['现售总套数']
|
||||||
|
row << sellObjMap["栋数"]
|
||||||
|
results << row
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ExcelUtils.write("D:\\数据\\新房\\${name}.xlsx", results, "深圳")
|
||||||
|
}
|
||||||
|
|
||||||
|
def getSellCodeObject(url){
|
||||||
|
String content = new HttpRequest().get(url, null, "utf-8")
|
||||||
|
def doc = Jsoup.parse(content);
|
||||||
|
Elements trs = doc.select("table tr")
|
||||||
|
def sellObjMap = [:]
|
||||||
|
def sellUseList = []
|
||||||
|
for(def i=1 ;i<trs.size();i++){
|
||||||
|
def tds = trs[i].select("td")
|
||||||
|
if(tds.size()==4){
|
||||||
|
sellObjMap.put(tds[0].text(), tds[1].text())
|
||||||
|
sellObjMap.put(tds[2].text(), tds[3].text())
|
||||||
|
}else if(tds.size()==6){
|
||||||
|
if(tds[1].text()!="--"){
|
||||||
|
def map = [:]
|
||||||
|
map.put(tds[0].text(), tds[1].text())
|
||||||
|
map.put(tds[2].text(), tds[3].text())
|
||||||
|
map.put(tds[4].text(), tds[5].text())
|
||||||
|
sellUseList << map
|
||||||
|
}
|
||||||
|
}else if(tds.size()==2){
|
||||||
|
sellObjMap.put(tds[0].text(), tds[1].text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sellObjMap.useList = sellUseList
|
||||||
|
return sellObjMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getProjectDetail(url){
|
||||||
|
String content = new HttpRequest().get(url, null, "utf-8")
|
||||||
|
def doc = Jsoup.parse(content);
|
||||||
|
// def sellInfoUrl = doc.select("strong:contains(卖方信息) a").attr("href")
|
||||||
|
// def devMap = getSellInfo("https://zjj.sz.gov.cn/ris/bol/szfdc/"+sellInfoUrl)
|
||||||
|
|
||||||
|
Elements tableEle = doc.select("table")
|
||||||
|
Elements trs1 = tableEle.get(0).select("tr")
|
||||||
|
def projMap = [:]
|
||||||
|
|
||||||
|
for(def i=1 ;i<trs1.size();i++) {
|
||||||
|
def tds = trs1[i].select("td")
|
||||||
|
|
||||||
|
for (def j = 0; j+2 <= tds.size(); j=j+2) {
|
||||||
|
projMap.put(tds[j].text(), tds[j + 1].text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Elements trs2 = tableEle.get(1).select("tr")
|
||||||
|
def projList = []
|
||||||
|
for(def i=1 ;i<trs2.size();i++) {
|
||||||
|
def tds = trs2[i].select("td")
|
||||||
|
def map = [:]
|
||||||
|
map.put("项目名称", tds[0].text())
|
||||||
|
map.put("楼名", tds[1].text())
|
||||||
|
map.put("建设工程规划许可证", tds[2].text())
|
||||||
|
map.put("建筑工程施工许可证", tds[3].text())
|
||||||
|
def roomUrl = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+tds[4].select("a").attr("href").replace("&","&")
|
||||||
|
map.put("套房信息", roomUrl)
|
||||||
|
|
||||||
|
|
||||||
|
def yusellroomList = getYuSellRoomDetail(roomUrl)
|
||||||
|
map.yusell_roomDetails = yusellroomList
|
||||||
|
|
||||||
|
// def nowsellroomList = getNowSellRoomDetail(roomUrl)
|
||||||
|
// map.nowsellroomDetails = nowsellroomList
|
||||||
|
projList << map
|
||||||
|
}
|
||||||
|
projMap.projList = projList
|
||||||
|
// projMap.devInfoMap = devMap
|
||||||
|
return projMap
|
||||||
|
}
|
||||||
|
|
||||||
|
def getYuSellRoomDetail(roomUrl){
|
||||||
|
String content = new HttpRequest().get(roomUrl, null, "utf-8")
|
||||||
|
def doc = Jsoup.parse(content);
|
||||||
|
def roomList = []
|
||||||
|
// Elements tdsEle = doc.select("td:contains(房号:)")
|
||||||
|
|
||||||
|
// for(def td : tdsEle){
|
||||||
|
// def tdText = td.text()
|
||||||
|
// if(tdText.contains("房号")){
|
||||||
|
// def roomMap = [:]
|
||||||
|
// def divEle = td.select("div")
|
||||||
|
// roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
|
||||||
|
// roomMap.put("房号状态", divEle.get(1).text())
|
||||||
|
// def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+divEle.get(1).select("a").attr("href")
|
||||||
|
// roomMap.put("房号url", href)
|
||||||
|
// roomMap = getRoomRefDetail(href, roomMap)
|
||||||
|
// roomList << roomMap
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
def hrefs = doc.select("#divShowBranch a")
|
||||||
|
for(def hrefObj : hrefs){
|
||||||
|
def danyuan = hrefObj.text().replace("[", "").replace("]", "")
|
||||||
|
def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/" + hrefObj.attr("href")
|
||||||
|
def danyuan_list = getDanyuanPage(href)
|
||||||
|
roomList.addAll(danyuan_list)
|
||||||
|
}
|
||||||
|
return roomList
|
||||||
|
}
|
||||||
|
|
||||||
|
def getDanyuanPage(roomUrl){
|
||||||
|
String content = new HttpRequest().get(roomUrl, null, "utf-8")
|
||||||
|
def doc = Jsoup.parse(content);
|
||||||
|
|
||||||
|
Elements tdsEle = doc.select("td:contains(房号:)")
|
||||||
|
def roomList = []
|
||||||
|
for(def td : tdsEle){
|
||||||
|
def tdText = td.text()
|
||||||
|
if(tdText.contains("房号")){
|
||||||
|
def roomMap = [:]
|
||||||
|
def divEle = td.select("div")
|
||||||
|
roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
|
||||||
|
roomMap.put("房号状态", divEle.get(1).text())
|
||||||
|
def href = divEle.get(1).select("a").attr("href")
|
||||||
|
|
||||||
|
if(href){
|
||||||
|
href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+ href
|
||||||
|
roomMap.put("房号url", href)
|
||||||
|
roomMap = getRoomRefDetail(href, roomMap)
|
||||||
|
}
|
||||||
|
roomList << roomMap
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return roomList
|
||||||
|
}
|
||||||
|
|
||||||
|
def getContent(href){
|
||||||
|
String content = null
|
||||||
|
def error = 0
|
||||||
|
while (error<=5){
|
||||||
|
try{
|
||||||
|
content = new HttpRequest().get(href, null, "utf-8")
|
||||||
|
if(content){
|
||||||
|
error = 100
|
||||||
|
}else{
|
||||||
|
Thread.sleep(3000)
|
||||||
|
error ++
|
||||||
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
println e.getMessage() + " "+ href
|
||||||
|
Thread.sleep(3000)
|
||||||
|
error ++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return content
|
||||||
|
}
|
||||||
|
|
||||||
|
def getRoomRefDetail(href, infoMap){
|
||||||
|
String content = getContent(href)
|
||||||
|
def doc = Jsoup.parse(content);
|
||||||
|
Elements trsEle = doc.select("table tr")
|
||||||
|
println href +" "+trsEle.size()
|
||||||
|
if(trsEle.size()==0){
|
||||||
|
def error = 0
|
||||||
|
while (error<=3){
|
||||||
|
Thread.sleep(3000)
|
||||||
|
content = new HttpRequest().get(href, null, "utf-8")
|
||||||
|
doc = Jsoup.parse(content);
|
||||||
|
trsEle = doc.select("table tr")
|
||||||
|
println href +" "+trsEle.size()
|
||||||
|
if(trsEle.size()>0){
|
||||||
|
error = 100
|
||||||
|
}else{
|
||||||
|
error ++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
for(def i=0;i<=1;i++){
|
||||||
|
def tds = trsEle[i].select("td")
|
||||||
|
for (def j = 0; j+2 <= tds.size(); j=j+2) {
|
||||||
|
infoMap.put(tds[j].text(), tds[j + 1].text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def tds1 = trsEle.get(2).select("td")
|
||||||
|
infoMap.put("拟售价格-按建筑面积计", tds1[1].text().replace("--元/平方米","").replace("(按建筑面积计)",""))
|
||||||
|
infoMap.put("拟售价格-按套内建筑面积计", tds1[2].text().replace("--元/平方米","").replace("(按套内建筑面积计)",""))
|
||||||
|
infoMap.put("合同号", tds1[4].text())
|
||||||
|
|
||||||
|
|
||||||
|
def tds2 = trsEle.get(4).select("td")
|
||||||
|
for (def j = 0; j+2 <= tds2.size(); j=j+2) {
|
||||||
|
infoMap.put("预售-"+tds2[j].text(), tds2[j + 1].text().replace("平方米",""))
|
||||||
|
}
|
||||||
|
|
||||||
|
def tds3 = trsEle.get(6).select("td")
|
||||||
|
for (def j = 0; j+2 <= tds3.size(); j=j+2) {
|
||||||
|
infoMap.put("竣工-"+tds3[j].text(), tds3[j + 1].text().replace("平方米",""))
|
||||||
|
}
|
||||||
|
return infoMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getSellInfo(url){
|
||||||
|
String content = new HttpRequest().get(url, null, "utf-8")
|
||||||
|
def doc = Jsoup.parse(content);
|
||||||
|
Elements trs1 = doc.select("table tr")
|
||||||
|
def devMap = [:]
|
||||||
|
for(def i=1 ;i<trs1.size();i++) {
|
||||||
|
def tds = trs1[i].select("td")
|
||||||
|
for (def j = 0; j+2 <= tds.size(); j=j+2) {
|
||||||
|
devMap.put(tds[j].text(), tds[j + 1].text())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return devMap
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getNowSellRoomDetail(url){
|
||||||
|
// def doc = Jsoup.parse(content);
|
||||||
|
// def header = [:]
|
||||||
|
// def VIEWSTATE = doc.select("#__VIEWSTATE").attr("value")
|
||||||
|
// def VIEWSTATEGENERATOR = doc.select("#__VIEWSTATEGENERATOR").attr("value")
|
||||||
|
// def EVENTVALIDATION = doc.select("#__EVENTVALIDATION").attr("value")
|
||||||
|
// doc.select("#__VIEWSTATEGENERATOR").attr("value")
|
||||||
|
// header.put("__VIEWSTATE", doc.select("#__VIEWSTATE").attr("value"))
|
||||||
|
// header.put("__EVENTTARGET", "imgBt2")
|
||||||
|
// header.put("__VIEWSTATEGENERATOR", doc.select("#__VIEWSTATEGENERATOR").attr("value"))
|
||||||
|
// header.put("__EVENTVALIDATION", doc.select("#__EVENTVALIDATION").attr("value"))
|
||||||
|
//// params.put("scriptManager1","updatepanel1|imgBt2")
|
||||||
|
//// String res = InvokeHttpRequest.post(url, params, "utf-8");
|
||||||
|
// getRoomNowSellTabInfo(url, VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
|
||||||
|
//
|
||||||
|
// def params = [:]
|
||||||
|
// params.put("id", "49636")
|
||||||
|
// params.put("presellid", "129765")
|
||||||
|
// HttpRequest re = new HttpRequest();
|
||||||
|
// String res = re.post(url, params, header, "utf-8");
|
||||||
|
|
||||||
|
String res = new HttpRequest().get(url+"&Branch=未命名&isBlock=xs", null, "utf-8")
|
||||||
|
def doc1 = Jsoup.parse(res);
|
||||||
|
|
||||||
|
Elements tdsEle = doc1.select("td:contains(房号:)")
|
||||||
|
def roomList = []
|
||||||
|
for(def td : tdsEle){
|
||||||
|
def tdText = td.text()
|
||||||
|
if(tdText.contains("房号")){
|
||||||
|
def roomMap = [:]
|
||||||
|
def divEle = td.select("div")
|
||||||
|
roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
|
||||||
|
roomMap.put("房号状态", divEle.get(1).text())
|
||||||
|
def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+divEle.get(1).select("a").attr("href")
|
||||||
|
roomMap.put("房号url", href)
|
||||||
|
|
||||||
|
roomMap = getRoomRefDetail(href, roomMap)
|
||||||
|
roomList << roomMap
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return roomList
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getRoomNowSellTabInfo(targetUrl, VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION){
|
||||||
|
try {
|
||||||
|
// 创建连接
|
||||||
|
URL url = new URL(targetUrl);
|
||||||
|
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||||
|
|
||||||
|
// 基础配置
|
||||||
|
conn.setRequestMethod("POST");
|
||||||
|
conn.setDoOutput(true); // 允许写入输出流
|
||||||
|
conn.setConnectTimeout(5000); // 5秒连接超时
|
||||||
|
conn.setReadTimeout(10000); // 10秒读取超时
|
||||||
|
|
||||||
|
// 设置请求头(模拟浏览器)
|
||||||
|
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
|
||||||
|
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
|
||||||
|
conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9");
|
||||||
|
|
||||||
|
// 构建Form Data参数(带中文需编码)
|
||||||
|
String formData = "__VIEWSTATE=" + URLEncoder.encode(VIEWSTATE, "UTF-8") +
|
||||||
|
"&__VIEWSTATEGENERATOR=" + URLEncoder.encode(VIEWSTATEGENERATOR, "UTF-8") +
|
||||||
|
"&__EVENTVALIDATION="+URLEncoder.encode(EVENTVALIDATION, "UTF-8") +
|
||||||
|
"&__EVENTTARGET=imgBt2";
|
||||||
|
|
||||||
|
// 发送请求体
|
||||||
|
OutputStream os = conn.getOutputStream()
|
||||||
|
byte[] input = formData.getBytes("utf-8");
|
||||||
|
os.write(input, 0, input.length);
|
||||||
|
|
||||||
|
|
||||||
|
// 获取响应状态码
|
||||||
|
int status = conn.getResponseCode();
|
||||||
|
System.out.println("响应状态码: " + status);
|
||||||
|
|
||||||
|
String responseLine;
|
||||||
|
// 读取响应内容
|
||||||
|
if (status == HttpURLConnection.HTTP_OK) {
|
||||||
|
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"))
|
||||||
|
StringBuilder response = new StringBuilder();
|
||||||
|
|
||||||
|
while ((responseLine = br.readLine()) != null) {
|
||||||
|
response.append(responseLine.trim());
|
||||||
|
}
|
||||||
|
System.out.println("响应内容:\n" + response.toString());
|
||||||
|
} else {
|
||||||
|
System.out.println("请求失败,错误码: " + status);
|
||||||
|
}
|
||||||
|
return responseLine
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
+64
-2
@@ -3,10 +3,12 @@ package com.fangjia.script.gj.data_handle
|
|||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
import com.fangjia.script.gj.ccb.FileUtils
|
import com.fangjia.script.gj.ccb.FileUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
|
import org.apache.commons.lang3.SerializationUtils
|
||||||
|
|
||||||
import java.util.regex.Pattern
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
@@ -14,15 +16,30 @@ class AddrAlaisProcess {
|
|||||||
|
|
||||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||||
|
//
|
||||||
|
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new AddrAlaisProcess()
|
def task = new AddrAlaisProcess()
|
||||||
|
|
||||||
// task.run()
|
// task.run()
|
||||||
// task.splitDist()
|
// task.splitDist()
|
||||||
// task.addAddr()
|
// task.addAddr()
|
||||||
task.nineApi()
|
// task.nineApi()
|
||||||
// task.createResult()
|
// task.createResult()
|
||||||
// task.mergeResult()
|
// task.mergeResult()
|
||||||
|
// task.addLocation()
|
||||||
|
task.updateAddr()
|
||||||
|
}
|
||||||
|
|
||||||
|
def addLocation(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\地图无路.xlsx",{d->
|
||||||
|
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||||
|
if(obj){
|
||||||
|
d['G'] = obj.lng/10000000000.0
|
||||||
|
d['H'] = obj.lat/10000000000.0
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
def splitDist(){
|
def splitDist(){
|
||||||
@@ -102,7 +119,7 @@ class AddrAlaisProcess {
|
|||||||
def nineApi(){
|
def nineApi(){
|
||||||
|
|
||||||
def i = 0;
|
def i = 0;
|
||||||
while(i<20) {
|
while(i<1) {
|
||||||
def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx"
|
def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx"
|
||||||
if (CommonDatautils.isWindow()) {
|
if (CommonDatautils.isWindow()) {
|
||||||
fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx"
|
fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx"
|
||||||
@@ -122,6 +139,16 @@ class AddrAlaisProcess {
|
|||||||
def roadObj = findBaiduDatas( data.city, d['H'])
|
def roadObj = findBaiduDatas( data.city, d['H'])
|
||||||
if(roadObj){
|
if(roadObj){
|
||||||
d['J'] = "百度存在"
|
d['J'] = "百度存在"
|
||||||
|
}else{
|
||||||
|
// def distData = mcaDistMap.get(d['D'] + "#" + d['E'])
|
||||||
|
// def fullDistName = null
|
||||||
|
// if (distData) {
|
||||||
|
// fullDistName = distData.district
|
||||||
|
// }
|
||||||
|
// def obj = CommonDatautils.findGaoDeDatas(data.city, fullDistName, d['H'])
|
||||||
|
// if(obj){
|
||||||
|
// d['J'] = "高德存在"
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -208,4 +235,39 @@ class AddrAlaisProcess {
|
|||||||
def obj = BAIDU_POI_COLL.findOne(q)
|
def obj = BAIDU_POI_COLL.findOne(q)
|
||||||
return obj;
|
return obj;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def updateAddr(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\地址及地址别名-详情-1.xlsx",{d->
|
||||||
|
def obj = CommonDatautils.findDistrictById(d['C'])
|
||||||
|
if(obj && d['G']){
|
||||||
|
def flag = DataHandleUtils.isStandardRoad(d['L'])
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
if(obj.address == d['H']){
|
||||||
|
udata.put("address", d['L'])
|
||||||
|
}else{
|
||||||
|
if(obj.addr_alias){
|
||||||
|
def new_addr_alias = SerializationUtils.clone(obj.addr_alias)
|
||||||
|
for(def alias : new_addr_alias){
|
||||||
|
if(d['H'] == alias){
|
||||||
|
if(flag){
|
||||||
|
if(d['L']!=alias){
|
||||||
|
alias = d['L']
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
alias = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(new_addr_alias!=obj.addr_alias){
|
||||||
|
udata.put("addr_alias", new_addr_alias)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(udata){
|
||||||
|
udata.put("m_date", new Date())
|
||||||
|
DbUtils.seaweed.update(new BasicDBObject('_id', obj._id),new BasicDBObject('$set',udata))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-5
@@ -225,15 +225,12 @@ class AutoAddOfficeBatch extends BaseBatch {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def repeatDist(){
|
def repeatDist(){
|
||||||
def citys = ["南京"]//CommonDatautils.findAllCity().reverse()
|
def citys = ["成都"]//CommonDatautils.findAllCity().reverse()
|
||||||
def allrets = []
|
def allrets = []
|
||||||
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
|
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
|
||||||
def index = 0
|
def index = 0
|
||||||
for (def c : citys) {
|
for (def c : citys) {
|
||||||
index++
|
index++
|
||||||
if(["宁波","无锡","青岛"].contains(c)){
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
|
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
|
||||||
def rets = checkRepeatDistrictBatch(c, districts)
|
def rets = checkRepeatDistrictBatch(c, districts)
|
||||||
println c + " " + districts.size()
|
println c + " " + districts.size()
|
||||||
@@ -254,13 +251,14 @@ class AutoAddOfficeBatch extends BaseBatch {
|
|||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
def cityId = cityCodeDataMap.get(city)
|
def cityId = cityCodeDataMap.get(city)
|
||||||
|
|
||||||
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502)
|
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202505)
|
||||||
def addCount = new AtomicLong()
|
def addCount = new AtomicLong()
|
||||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||||
def total = districts.size()
|
def total = districts.size()
|
||||||
def rets = new CopyOnWriteArrayList()
|
def rets = new CopyOnWriteArrayList()
|
||||||
def sets = new CopyOnWriteArraySet()
|
def sets = new CopyOnWriteArraySet()
|
||||||
for (def i = 0; i < total; i++) {
|
for (def i = 0; i < total; i++) {
|
||||||
|
println i
|
||||||
final def o1 = districts[i]
|
final def o1 = districts[i]
|
||||||
// fixedThreadPool.execute(new Runnable() {
|
// fixedThreadPool.execute(new Runnable() {
|
||||||
// void run() {
|
// void run() {
|
||||||
|
|||||||
+86
@@ -0,0 +1,86 @@
|
|||||||
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
|
class CommunityShapeProcess {
|
||||||
|
|
||||||
|
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
|
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new CommunityShapeProcess();
|
||||||
|
task.run();
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("province_name", "河北省")
|
||||||
|
q.put("shape", new BasicDBObject('$exists', true))
|
||||||
|
def keys = new BasicDBObject()
|
||||||
|
keys.put("shape", 1)
|
||||||
|
def dbcursor = community_CONN.find(q, keys).batchSize(1000)
|
||||||
|
for(def d : dbcursor){
|
||||||
|
if(d.shape.type=="Polygon"){
|
||||||
|
def coordinates = d.shape.coordinates.get(0)
|
||||||
|
def points = []
|
||||||
|
def index = 0
|
||||||
|
for(def arrs : coordinates){
|
||||||
|
if(index!=0 && index+1!= coordinates.size()){
|
||||||
|
if(!points.contains(arrs)){
|
||||||
|
points << arrs
|
||||||
|
}else{
|
||||||
|
println d._id +" 重复数据:"+arrs
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
points << arrs
|
||||||
|
}
|
||||||
|
index++
|
||||||
|
}
|
||||||
|
if(points.size()!=coordinates.size()){
|
||||||
|
d.shape.coordinates = [points]
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.shape = d.shape
|
||||||
|
community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println d._id
|
||||||
|
def coordinateArrs = d.shape.coordinates
|
||||||
|
def multiPoints = []
|
||||||
|
def updateFlag = false
|
||||||
|
for(def typeCoordinates : coordinateArrs){
|
||||||
|
def pointArrs = []
|
||||||
|
|
||||||
|
for(def coordinates : typeCoordinates){
|
||||||
|
def points = []
|
||||||
|
def index = 0
|
||||||
|
for(def arrs : coordinates){
|
||||||
|
if(index!=0 && index+1!= coordinates.size()){
|
||||||
|
if(!points.contains(arrs)){
|
||||||
|
points << arrs
|
||||||
|
}else{
|
||||||
|
println d._id +" 重复数据:"+arrs
|
||||||
|
updateFlag = true
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
points << arrs
|
||||||
|
}
|
||||||
|
index++
|
||||||
|
}
|
||||||
|
pointArrs << points
|
||||||
|
}
|
||||||
|
multiPoints << pointArrs
|
||||||
|
}
|
||||||
|
|
||||||
|
if(updateFlag){
|
||||||
|
d.shape.coordinates = multiPoints
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.shape = d.shape
|
||||||
|
community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+17
-6
@@ -93,12 +93,23 @@ class DevProcess {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def aiDevTown(){
|
def aiDevTown(){
|
||||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
|
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
|
||||||
def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
|
// def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
|
||||||
def message = res.data.replaceAll("\n", "")
|
// def message = res.data.replaceAll("\n", "")
|
||||||
d['F'] = res.code
|
// d['F'] = res.code
|
||||||
d['G'] = message
|
// d['G'] = message
|
||||||
println d['E'] +" "+res.code + " "+message
|
// println d['E'] +" "+res.code + " "+message
|
||||||
|
def q = new BasicDBObject("name", d['E']).append("type","乡镇街道")
|
||||||
|
def dbcursor = DEV_MAPPING_CONN.find(q)
|
||||||
|
if(dbcursor.count()>0){
|
||||||
|
def list = []
|
||||||
|
for(def db : dbcursor){
|
||||||
|
list << db.township
|
||||||
|
}
|
||||||
|
|
||||||
|
d['H']= list.join(",")
|
||||||
|
}
|
||||||
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+75
@@ -0,0 +1,75 @@
|
|||||||
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.lixue.util.DBUtils
|
||||||
|
import com.fangjia.script.looplineinfo.InPolygon
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
|
import java.awt.geom.Point2D
|
||||||
|
|
||||||
|
class DistrictLoopInfo {
|
||||||
|
|
||||||
|
DBCollection LATLNG_POLYGON_COLL = MongoUtils.getCollectionByName("fangjia", "latlng_polygon", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new DistrictLoopInfo()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("name", new BasicDBObject('$in', ["内环","中环","外环","郊环"]))
|
||||||
|
def loopShapes = LATLNG_POLYGON_COLL.find(q)
|
||||||
|
def loopMap = [:]
|
||||||
|
for(def loop : loopShapes){
|
||||||
|
loopMap.put(loop.name, loop)
|
||||||
|
}
|
||||||
|
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city", "region", "name", "lat", "lng", "loop_line_info"])
|
||||||
|
println districts.size()
|
||||||
|
def index = 0
|
||||||
|
for(def d : districts){
|
||||||
|
index++
|
||||||
|
if(d.lng){
|
||||||
|
def lng = d.lng/10000000000.0
|
||||||
|
def lat = d.lat/10000000000.0
|
||||||
|
Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6, lng), DataHandleUtils.getDoubleRound(6, lat));
|
||||||
|
def loop1 = loopMap.get("内环")
|
||||||
|
def flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
|
||||||
|
def loop_line_info = ""
|
||||||
|
if(flag){
|
||||||
|
loop_line_info = "内环内"
|
||||||
|
}else{
|
||||||
|
loop1 = loopMap.get("中环")
|
||||||
|
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
|
||||||
|
if(flag){
|
||||||
|
loop_line_info = "内中环"
|
||||||
|
}else{
|
||||||
|
loop1 = loopMap.get("外环")
|
||||||
|
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
|
||||||
|
if(flag){
|
||||||
|
loop_line_info = "中外环"
|
||||||
|
}else{
|
||||||
|
loop1 = loopMap.get("郊环")
|
||||||
|
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
|
||||||
|
if(flag){
|
||||||
|
loop_line_info = "外郊环"
|
||||||
|
}else{
|
||||||
|
loop_line_info = "郊环外"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println index +" "+d.region +" "+d.name+" 原:"+d.loop_line_info+" 现:"+loop_line_info+" lng:"+lng+","+lat
|
||||||
|
def udata =new BasicDBObject()
|
||||||
|
udata.put("loop_line_info", loop_line_info)
|
||||||
|
DBUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set',udata))
|
||||||
|
}else{
|
||||||
|
println index + " "+d.region +" "+d.name+"无经纬度信息"
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+41
-1
@@ -22,7 +22,47 @@ class DistrictRepeatBatch {
|
|||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new DistrictRepeatBatch()
|
def task = new DistrictRepeatBatch()
|
||||||
// task.run()
|
// task.run()
|
||||||
task.resultProcess()
|
// task.resultProcess()
|
||||||
|
task.processRepeatAndAlias()
|
||||||
|
}
|
||||||
|
|
||||||
|
def processRepeatAndAlias(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\小区\\重复小区\\结果\\重复可疑小区-南京.xlsx",{d->
|
||||||
|
if(d['H'] && d['H'] == "删除"){
|
||||||
|
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||||
|
if(!obj){
|
||||||
|
obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||||
|
}
|
||||||
|
if(obj){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("status", 1)
|
||||||
|
udata.put("m_date", new Date())
|
||||||
|
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||||
|
saveOptLog(obj.city, obj.region, obj.name, "remove")
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(d['I']){
|
||||||
|
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||||
|
if(obj){
|
||||||
|
def arrs = d['I'].split(",")
|
||||||
|
if(arrs){
|
||||||
|
def alias = obj.alias
|
||||||
|
if(!alias){
|
||||||
|
alias = []
|
||||||
|
}
|
||||||
|
alias.addAll(arrs)
|
||||||
|
alias = alias.unique()
|
||||||
|
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("m_date", new Date())
|
||||||
|
udata.put("alias", alias)
|
||||||
|
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||||
|
saveOptLog(obj.city, obj.region, obj.name, "update")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},0)
|
||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
|
|||||||
+1
-1
@@ -17,7 +17,7 @@ import com.fangjia.script.util.OpenApiClient
|
|||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
class GXDdistrictMatch {
|
class 8GXDdistrictMatch {
|
||||||
|
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
|
|
||||||
|
|||||||
+137
-6
@@ -29,8 +29,8 @@ class TownMiddleProcess {
|
|||||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||||
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
|
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping_0526", MONGO100_URL)
|
||||||
|
DBCollection TEMP_ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
|
||||||
|
|
||||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
@@ -65,7 +65,7 @@ class TownMiddleProcess {
|
|||||||
//修改乡镇街道别名
|
//修改乡镇街道别名
|
||||||
// updateTownAlias()
|
// updateTownAlias()
|
||||||
//中间表同步到100
|
//中间表同步到100
|
||||||
synchTown()
|
// synchTown()
|
||||||
//添加地级市标志
|
//添加地级市标志
|
||||||
// addCityFlag()
|
// addCityFlag()
|
||||||
// addPinyin()
|
// addPinyin()
|
||||||
@@ -101,11 +101,22 @@ class TownMiddleProcess {
|
|||||||
|
|
||||||
// updateNewRegionPoyline()//更新新的区域围栏
|
// updateNewRegionPoyline()//更新新的区域围栏
|
||||||
|
|
||||||
updateMappingPolyline()
|
// updateMappingPolyline()
|
||||||
|
|
||||||
|
//批量修改多边型围栏数据结构
|
||||||
|
// updatePolylineBatch()
|
||||||
|
// test2dshape()
|
||||||
|
|
||||||
|
//将省会城市更新到数据库中
|
||||||
|
// updateProvinceCenter()
|
||||||
|
//城市级别更新到库中
|
||||||
|
// updateCityLevel()
|
||||||
|
//城市顺序
|
||||||
|
// updateCityNum()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def createProvince(){
|
def createProvince(){
|
||||||
def cat = "省"
|
def cat = "省"
|
||||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||||
@@ -457,16 +468,17 @@ class TownMiddleProcess {
|
|||||||
def datas = []
|
def datas = []
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
datas << d
|
datas << d
|
||||||
if (datas.size() >= 2000) {
|
if (datas.size() >= 5000) {
|
||||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||||
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||||
|
// ADMINISTRATIVE_MAPPING_COLL.insert(datas)
|
||||||
println datas.size()
|
println datas.size()
|
||||||
datas.clear();
|
datas.clear();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (datas.size() >= 0) {
|
if (datas.size() >= 0) {
|
||||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
ADMINISTRATIVE_MAPPING_COLL.insert(datas)
|
||||||
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||||
datas.clear();
|
datas.clear();
|
||||||
}
|
}
|
||||||
@@ -1449,4 +1461,123 @@ class TownMiddleProcess {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def updatePolylineBatch(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("shape",new BasicDBObject('$exists', true))
|
||||||
|
q.put("update_shape",new BasicDBObject('$exists', false))
|
||||||
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100)
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def type = d.shape.type
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
def shapeMap = [:]
|
||||||
|
if(type == "polygon"){
|
||||||
|
shapeMap.put("type", "Polygon")
|
||||||
|
shapeMap.put("coordinates", d.shape.coordinates)
|
||||||
|
udata.put("shape", shapeMap)
|
||||||
|
udata.put("update_shape", 1)
|
||||||
|
ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata))
|
||||||
|
}else if(type == "MultiPolygon"){
|
||||||
|
def coordinates = d.shape.coordinates
|
||||||
|
def new_coors = []
|
||||||
|
for(def coors : coordinates){
|
||||||
|
new_coors << [coors]
|
||||||
|
}
|
||||||
|
shapeMap.put("type", "MultiPolygon")
|
||||||
|
shapeMap.put("coordinates", new_coors)
|
||||||
|
udata.put("shape", shapeMap)
|
||||||
|
udata.put("update_shape", 1)
|
||||||
|
ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def test2dshape(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("street","花亭路街道")
|
||||||
|
// q.put("shape",new BasicDBObject('$exists', true))
|
||||||
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100)
|
||||||
|
for(def obj : dbcursor){
|
||||||
|
try{
|
||||||
|
TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj)
|
||||||
|
}catch(Exception e){
|
||||||
|
def coordinates = obj.shape.coordinates
|
||||||
|
for(def coors : coordinates){
|
||||||
|
for (int i = 0; i < coors.size(); i++) { // 改为索引遍历
|
||||||
|
def coorsArr = coors[i]
|
||||||
|
def index = 0
|
||||||
|
def new_coors = []
|
||||||
|
for(def points : coorsArr){
|
||||||
|
index ++
|
||||||
|
if(!new_coors.contains(points)){
|
||||||
|
new_coors << points
|
||||||
|
}else{
|
||||||
|
if(index<=coorsArr.size()){
|
||||||
|
new_coors << points
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(new_coors.size()!=coorsArr.size()){
|
||||||
|
coors[i] = new_coors
|
||||||
|
println "[修改] "+ obj.city +" "+obj.street +" 原:"+coorsArr.size()+" 现:"+new_coors.size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try{
|
||||||
|
TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj)
|
||||||
|
}catch(Exception ex){
|
||||||
|
println "仍有问题:"+obj.city +" "+obj.street+" "+ex.getMessage()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def updateProvinceCenter(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\省会城市.xlsx",{d->
|
||||||
|
def city = d['C']
|
||||||
|
if(city){
|
||||||
|
def q = new BasicDBObject("type","城市").append("city_short", city)
|
||||||
|
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||||
|
if(obj){
|
||||||
|
def udata = new BasicDBObject("capital_city", "是")
|
||||||
|
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
def updateCityLevel(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市等级.xlsx",{d->
|
||||||
|
def city = d['A']
|
||||||
|
if(city){
|
||||||
|
def q = new BasicDBObject("type","城市").append("city_short", city)
|
||||||
|
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||||
|
if(obj){
|
||||||
|
def udata = new BasicDBObject("city_level", d['B'])
|
||||||
|
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
|
||||||
|
}else{
|
||||||
|
println city +" 没找到"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
def updateCityNum(){
|
||||||
|
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市排序.xlsx",{d->
|
||||||
|
def city = d['A']
|
||||||
|
if(city){
|
||||||
|
def q = new BasicDBObject("type","城市").append("city_short", city)
|
||||||
|
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||||
|
if(obj){
|
||||||
|
def udata = new BasicDBObject("city_position", d['B'].toString().toDouble().toInteger())
|
||||||
|
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
|
||||||
|
}else{
|
||||||
|
println city +" 没找到"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-2
@@ -18,16 +18,17 @@ class GaodeCityRange {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def citys = CommonDatautils.findAllCity()
|
// def citys = CommonDatautils.findAllCity()
|
||||||
|
|
||||||
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
|
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
|
||||||
|
def citys = ["新星","胡杨河","白杨"]
|
||||||
HttpRequest request = new HttpRequest()
|
HttpRequest request = new HttpRequest()
|
||||||
def rows = []
|
def rows = []
|
||||||
|
|
||||||
citys.each{c->
|
citys.each{c->
|
||||||
if(!citys1.contains(c)){
|
if(!citys1.contains(c)){
|
||||||
c = c.replace("州","")
|
c = c.replace("州","")
|
||||||
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
|
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=66e6771767ab293ace441553c36621c1&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
|
||||||
if(resp.containsKey("districts")){
|
if(resp.containsKey("districts")){
|
||||||
def datas = resp.get("districts")
|
def datas = resp.get("districts")
|
||||||
for(def d : datas){
|
for(def d : datas){
|
||||||
|
|||||||
+134
-11
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.location
|
|||||||
|
|
||||||
import com.fangjia.core.RegionData
|
import com.fangjia.core.RegionData
|
||||||
import com.fangjia.core.util.HttpRequest
|
import com.fangjia.core.util.HttpRequest
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.Bytes
|
import com.mongodb.Bytes
|
||||||
@@ -22,6 +23,11 @@ class RegionPointCrawl {
|
|||||||
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||||
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||||
|
|
||||||
|
static DBCollection TOWN_CENTERCOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_center")
|
||||||
|
|
||||||
|
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new RegionPointCrawl();
|
def task = new RegionPointCrawl();
|
||||||
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
|
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
|
||||||
@@ -40,8 +46,12 @@ class RegionPointCrawl {
|
|||||||
// println "区域围栏 完成"
|
// println "区域围栏 完成"
|
||||||
// }
|
// }
|
||||||
|
|
||||||
task.statSame()
|
// task.statSame()
|
||||||
|
|
||||||
|
//"福建省","河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","山东省","湖北省","湖南省","江西省","江苏省","安徽省"
|
||||||
|
def provinces = ["河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","青海省","山东省","福建省","浙江省","台湾省","河南省","湖北省","湖南省","江西省","江苏省","安徽省","广东省","海南省","四川省","贵州省","云南省","北京市","上海市","天津市","重庆市","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||||
|
task.getTownshipCenter(provinces)
|
||||||
|
task.addTownCenter(provinces)
|
||||||
}
|
}
|
||||||
|
|
||||||
def crawlCityAndDistrictCode(def province){
|
def crawlCityAndDistrictCode(def province){
|
||||||
@@ -210,7 +220,7 @@ class RegionPointCrawl {
|
|||||||
jo = JSONObject.fromObject(html)
|
jo = JSONObject.fromObject(html)
|
||||||
if(jo.infocode!="10000"){
|
if(jo.infocode!="10000"){
|
||||||
errorCount ++
|
errorCount ++
|
||||||
Thread.sleep(30*1000)
|
Thread.sleep(3*1000)
|
||||||
// println " 不可用Key: "+key
|
// println " 不可用Key: "+key
|
||||||
}else{
|
}else{
|
||||||
errorCount = 100
|
errorCount = 100
|
||||||
@@ -228,15 +238,15 @@ class RegionPointCrawl {
|
|||||||
|
|
||||||
static String getRandomKey2() {
|
static String getRandomKey2() {
|
||||||
def KeyList = [
|
def KeyList = [
|
||||||
"2af0b8533415a612b98d59bb93d664b9",
|
// "2af0b8533415a612b98d59bb93d664b9",
|
||||||
"0246c57bbcf24aa246c207b3c99dfe41",
|
// "0246c57bbcf24aa246c207b3c99dfe41",
|
||||||
"042328b3f0fa5daf426802d379521d87",
|
// "042328b3f0fa5daf426802d379521d87",
|
||||||
"055d4f1b5b64a2bce18f4fe1f15c2639",
|
// "055d4f1b5b64a2bce18f4fe1f15c2639",
|
||||||
"d5484f5170502405d88bbaffe1b496f0",
|
// "d5484f5170502405d88bbaffe1b496f0",
|
||||||
"e74ccc4258079d2200199695110cceb1",
|
// "e74ccc4258079d2200199695110cceb1",
|
||||||
"f58629fc799fbc8537ca85046c6d44a5",
|
// "f58629fc799fbc8537ca85046c6d44a5",
|
||||||
"291073f17ee0b963ccb1927ed92bf265",
|
"eb69e4ad2a9cdd31f163a0ed32effe6e"
|
||||||
"2a1fb75990f2b39f6221e50892dd3988"
|
// "2af0b8533415a612b98d59bb93d664b9"
|
||||||
]
|
]
|
||||||
|
|
||||||
def count = RandomUtils.nextInt(KeyList.size())
|
def count = RandomUtils.nextInt(KeyList.size())
|
||||||
@@ -300,4 +310,117 @@ class RegionPointCrawl {
|
|||||||
}
|
}
|
||||||
return map;
|
return map;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getTownshipCenter(provinces){
|
||||||
|
for(def p : provinces){
|
||||||
|
def query = new BasicDBObject("province", p)
|
||||||
|
def count = TOWN_CENTERCOLL.count(query)
|
||||||
|
if(count){
|
||||||
|
println p +" 已存在"
|
||||||
|
}else{
|
||||||
|
def ret = polygon(p, "3", "base");
|
||||||
|
if(ret){
|
||||||
|
def citys = ret[0].districts
|
||||||
|
if(citys){
|
||||||
|
def allRets = []
|
||||||
|
for(def c : citys){
|
||||||
|
def cityTowns = getTownByCity(p, c.name,c.districts)
|
||||||
|
allRets.addAll(cityTowns)
|
||||||
|
}
|
||||||
|
saveTownCenter(allRets)
|
||||||
|
println p +" 保存:"+allRets.size()
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println p+ "无结果"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def getTownByCity(prov, city,distDatas){
|
||||||
|
def rets = []
|
||||||
|
for(def d : distDatas){
|
||||||
|
def townDatas = d.districts
|
||||||
|
for(def town : townDatas){
|
||||||
|
town.province = prov
|
||||||
|
town.district = d.name
|
||||||
|
town.city = city
|
||||||
|
rets << town
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rets
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def saveTownCenter(allRets){
|
||||||
|
for(def r : allRets){
|
||||||
|
def obj = new BasicDBObject()
|
||||||
|
obj.put("province", r.province)
|
||||||
|
obj.put("city", r.city)
|
||||||
|
obj.put("district", r.district)
|
||||||
|
obj.put("town", r.name)
|
||||||
|
obj.put("center", r.center)
|
||||||
|
obj.put("level", r.level)
|
||||||
|
TOWN_CENTERCOLL.save(obj)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def addTownCenter(provinces){
|
||||||
|
for(def p : provinces){
|
||||||
|
def q = new BasicDBObject("province",p).append("type","乡镇街道")
|
||||||
|
// q.put("location", new BasicDBObject('$exists', false))
|
||||||
|
def dbcursor = administrative_mapping_coll.find(q)
|
||||||
|
for(def obj : dbcursor){
|
||||||
|
|
||||||
|
def tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", obj.street)
|
||||||
|
def dbo = TOWN_CENTERCOLL.findOne(tq)
|
||||||
|
if(!dbo){
|
||||||
|
def new_street = obj.street.replace("管理处","")
|
||||||
|
tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", new_street)
|
||||||
|
dbo = TOWN_CENTERCOLL.findOne(tq)
|
||||||
|
}
|
||||||
|
if(dbo){
|
||||||
|
def arrs = dbo.center.split(",")
|
||||||
|
def location = [:]
|
||||||
|
location["lon"] = arrs[0].toDouble()
|
||||||
|
location["lat"] = arrs[1].toDouble()
|
||||||
|
if(!obj.location){
|
||||||
|
def udata = new BasicDBObject("location", location)
|
||||||
|
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||||
|
println("【修改】"+obj.city +" "+ obj.district+" "+obj.street)
|
||||||
|
}else{
|
||||||
|
def distance = com.fangjia.script.util.GeoUtils.calculateDistance(location.lat, location.lon, obj.location.lat, obj.location.lon);
|
||||||
|
if(distance>1000){
|
||||||
|
def udata = new BasicDBObject("location", location)
|
||||||
|
println(obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 经纬度中心点相距1公里以外 原:"+obj.location.lon+","+obj.location.lat+" 新:"+dbo.center+" 距离:"+distance)
|
||||||
|
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
|
||||||
|
def esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street)
|
||||||
|
if(!esobj){
|
||||||
|
def short_prov = obj.province.replace("省", "")
|
||||||
|
if(obj.street.contains(short_prov)){
|
||||||
|
esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street.replace(short_prov, ""))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(esobj){
|
||||||
|
|
||||||
|
def location = [:]
|
||||||
|
location["lon"] = esobj.location.lon.toDouble()
|
||||||
|
location["lat"] = esobj.location.lat.toDouble()
|
||||||
|
def udata = new BasicDBObject("location", location)
|
||||||
|
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||||
|
println("【修改】"+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street+" "+ esobj.cat+" "+esobj.name +" es poi ")
|
||||||
|
}else{
|
||||||
|
println(obj.province+" "+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 没有找到")
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+15
@@ -0,0 +1,15 @@
|
|||||||
|
package com.fangjia.script.gj.test
|
||||||
|
|
||||||
|
class NonghangDataExport {
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new NonghangDataExport();
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
def run(){
|
||||||
|
def citys = ["ÁÙÒÊ","¹ãÖÝ", "ÉîÛÚ"]
|
||||||
|
for(def c : citys){
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+4
-3
@@ -2,6 +2,7 @@ package com.fangjia.script.linxsh.util
|
|||||||
|
|
||||||
import com.fangjia.core.RegionData
|
import com.fangjia.core.RegionData
|
||||||
import com.fangjia.script.util.DataUtil
|
import com.fangjia.script.util.DataUtil
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.mongodb.DBCursor
|
import com.mongodb.DBCursor
|
||||||
@@ -47,6 +48,8 @@ class AddressUtil {
|
|||||||
|
|
||||||
private static final cityMap = new HashMap<String, String>()
|
private static final cityMap = new HashMap<String, String>()
|
||||||
|
|
||||||
|
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
static {
|
static {
|
||||||
initCityMap()
|
initCityMap()
|
||||||
}
|
}
|
||||||
@@ -55,9 +58,7 @@ class AddressUtil {
|
|||||||
* 初始化城市数据
|
* 初始化城市数据
|
||||||
*/
|
*/
|
||||||
static def initCityMap() {
|
static def initCityMap() {
|
||||||
Mongo mongo = new Mongo("mongo.db.fangjia.com", 28017);
|
DBCursor cursor = SERVER_AREA_COLL.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1));
|
||||||
DBCollection SERVER_AREA = mongo.getDB("fangjia").getCollection("server_area");
|
|
||||||
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1));
|
|
||||||
for(DBObject dbObject : cursor) {
|
for(DBObject dbObject : cursor) {
|
||||||
if(!"全国".equals(dbObject.display)) {
|
if(!"全国".equals(dbObject.display)) {
|
||||||
cityMap.put(dbObject.display, dbObject.display)
|
cityMap.put(dbObject.display, dbObject.display)
|
||||||
|
|||||||
@@ -239,7 +239,7 @@ public class CommonUtils {
|
|||||||
println(cmd)
|
println(cmd)
|
||||||
DBObject data = new BasicDBObject()
|
DBObject data = new BasicDBObject()
|
||||||
data.put("category", "sell")
|
data.put("category", "sell")
|
||||||
data.put("priority", 3)
|
data.put("priority", 1)
|
||||||
data.put("status", 0)
|
data.put("status", 0)
|
||||||
data.put("need_redraw_trend", needRedrawTrend)
|
data.put("need_redraw_trend", needRedrawTrend)
|
||||||
data.put("remark", 'week')
|
data.put("remark", 'week')
|
||||||
|
|||||||
@@ -1548,6 +1548,34 @@ public class OpenApiClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static JSONObject houseAction(String city, String address) {
|
||||||
|
try {
|
||||||
|
Map<String, String> params = new HashMap<String, String>();
|
||||||
|
params.put("city", city);
|
||||||
|
params.put("address", address);
|
||||||
|
params.put("token", TOKEN);
|
||||||
|
params.put("__refresh_cache","1")
|
||||||
|
|
||||||
|
String content = new HttpRequest().get("https://open.fangjia.com/house/around", params, "utf-8")
|
||||||
|
JSONObject result = JSONObject.fromObject(content);
|
||||||
|
while (result == null) {
|
||||||
|
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/house/around", params, "utf-8")
|
||||||
|
println("${city} ${address} ³ÌÐòµ÷ÓÃÒì³£")
|
||||||
|
Thread.sleep(6000)
|
||||||
|
}
|
||||||
|
if (result.getInt("code") == 0) {
|
||||||
|
return result.getJSONObject("result");
|
||||||
|
} else if (result.getInt("code") !=0) {
|
||||||
|
println("${city} ${address}")
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
println("${city} ${address} ${e.getMessage()}")
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-7
@@ -21,15 +21,15 @@ public class DistrictMonthlyCalc{
|
|||||||
|
|
||||||
|
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
List<String> cities = ["ÉϺ£"];
|
List<String> cities = ["ÏåÑô"];
|
||||||
def allCity = CommonUtils.findAllCity()
|
// def allCity = CommonUtils.findAllCity()
|
||||||
cities.addAll(allCity)
|
// cities.addAll(allCity)
|
||||||
def districts = []
|
def districts = []
|
||||||
cities.each {def city->
|
cities.each {def city->
|
||||||
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("name", 1).append("region", 1)).batchSize(2000).toList()
|
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("name", 1).append("region", 1)).batchSize(2000).toList()
|
||||||
println(city)
|
println(city)
|
||||||
}
|
}
|
||||||
ExecutorService executorService = Executors.newFixedThreadPool(20)
|
ExecutorService executorService = Executors.newFixedThreadPool(1)
|
||||||
districts.each {BasicDBObject district->
|
districts.each {BasicDBObject district->
|
||||||
executorService.submit(new Calculator(district))
|
executorService.submit(new Calculator(district))
|
||||||
}
|
}
|
||||||
@@ -43,7 +43,7 @@ public class DistrictMonthlyCalc{
|
|||||||
class Calculator extends Thread{
|
class Calculator extends Thread{
|
||||||
|
|
||||||
static DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
static DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||||
static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat_new", "mongo.db.fangjia.com")
|
||||||
final BasicDBObject district;
|
final BasicDBObject district;
|
||||||
|
|
||||||
public Calculator(final BasicDBObject district){
|
public Calculator(final BasicDBObject district){
|
||||||
@@ -52,14 +52,22 @@ class Calculator extends Thread{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
void run() {
|
void run() {
|
||||||
REGION_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo->
|
def q = new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)
|
||||||
|
q.put("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01", "yyyy-MM-dd")))
|
||||||
|
REGION_STAT.find(q).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo->
|
||||||
Date weekend = dbo.weekend
|
Date weekend = dbo.weekend
|
||||||
if(DateUtil.isLastSundayOfMonth(weekend)){
|
if(DateUtil.isLastSundayOfMonth(weekend)){
|
||||||
def trend = new BasicDBObject("city", district.city)
|
def trend = new BasicDBObject("city", district.city)
|
||||||
trend.append("region", district.region)
|
trend.append("region", district.region)
|
||||||
trend.append("district_name", district.name)
|
trend.append("district_name", district.name)
|
||||||
trend.append("date", weekend.format("yyyyMM").toInteger())
|
trend.append("date", weekend.format("yyyyMM").toInteger())
|
||||||
REGION_STAT_TREND.update(trend , new BasicDBObject('$set', new BasicDBObject("avg_price", dbo.avg_price)),true,false)
|
def udata = new BasicDBObject()
|
||||||
|
if(dbo.avg_price_2b && dbo.value_type!=1){
|
||||||
|
udata.put("avg_price", dbo.avg_price_2b)
|
||||||
|
}else{
|
||||||
|
udata.put("avg_price", dbo.avg_price)
|
||||||
|
}
|
||||||
|
REGION_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
REGION_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("data", 1)).each {BasicDBObject trend->
|
REGION_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("data", 1)).each {BasicDBObject trend->
|
||||||
|
|||||||
Reference in New Issue
Block a user