Merge branch 'script_gj' into 'master'
Script gj See merge request !1149
This commit is contained in:
+16
-12
@@ -29,7 +29,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
|
||||
@Override
|
||||
void process(Object args) {
|
||||
List<String> cities = [];
|
||||
def allCity = CommonUtils.findAllCity()
|
||||
def allCity = ["广州","深圳","临沂"]//CommonUtils.findAllCity()
|
||||
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||
cities.addAll(allCity)
|
||||
@@ -38,7 +38,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
|
||||
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList()
|
||||
println(city)
|
||||
}
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(20)
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(1)
|
||||
districts.each {BasicDBObject district->
|
||||
executorService.submit(new Calculator(district))
|
||||
}
|
||||
@@ -65,22 +65,26 @@ class Calculator extends Thread{
|
||||
}
|
||||
def statMap = [:]
|
||||
|
||||
DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo->
|
||||
DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type).append("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01","yyyy-MM-dd")))).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo->
|
||||
Date weekend = dbo.weekend
|
||||
if(DateUtil.isLastSundayOfMonth(weekend)){
|
||||
def trend = new BasicDBObject("city", district.city)
|
||||
trend.append("region", district.region)
|
||||
trend.append("district_name", district.name)
|
||||
trend.append("estate_type", estate_type)
|
||||
// trend.append("estate_type", estate_type)
|
||||
trend.append("date", weekend.format("yyyyMM").toInteger())
|
||||
|
||||
|
||||
def udata = new BasicDBObject();
|
||||
udata.put("avg_price", dbo.avg_price)
|
||||
udata.put("credit", dbo.credit)
|
||||
udata.put("m_date", new Date())
|
||||
DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
|
||||
statMap.put(weekend.format("yyyyMM").toInteger(), udata)
|
||||
if(dbo.avg_price_2b){
|
||||
// udata.put("avg_price", dbo.avg_price)
|
||||
udata.put("avg_price", dbo.avg_price_2b)
|
||||
udata.put("credit", dbo.credit)
|
||||
udata.put("m_date", new Date())
|
||||
DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
|
||||
statMap.put(weekend.format("yyyyMM").toInteger(), udata)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
DISTRICT_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).sort(new BasicDBObject("date", 1)).each {BasicDBObject trend->
|
||||
@@ -88,14 +92,14 @@ class Calculator extends Thread{
|
||||
def monDate = getMomDate(date)
|
||||
def yoyDate = getYoyDate(date)
|
||||
BasicDBObject monDbo = statMap.get(monDate)
|
||||
if(monDbo != null && monDbo.avg_price != 0) {
|
||||
trend.mom = DataUtil.roundDouble(((trend.avg_price - monDbo.avg_price)/(double)monDbo.avg_price),4)
|
||||
if(monDbo != null && monDbo.avg_price_2b != 0) {
|
||||
trend.mom = DataUtil.roundDouble(((trend.avg_price_2b - monDbo.avg_price_2b)/(double)monDbo.avg_price_2b),4)
|
||||
}else{
|
||||
trend.mom = 0d
|
||||
}
|
||||
BasicDBObject yoyDbo = statMap.get(yoyDate)
|
||||
if(yoyDbo != null && yoyDbo.avg_price != 0){
|
||||
trend.yoy = DataUtil.roundDouble(((trend.avg_price - yoyDbo.avg_price)/(double)yoyDbo.avg_price),4)
|
||||
trend.yoy = DataUtil.roundDouble(((trend.avg_price_2b - yoyDbo.avg_price_2b)/(double)yoyDbo.avg_price_2b),4)
|
||||
}else{
|
||||
trend.yoy = 0d
|
||||
}
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年5月深圳成交明细.xlsx", 0, 1)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
+26
@@ -855,6 +855,32 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictsKey(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
// q.put("status",new BasicDBObject('$ne', 0))
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
// q.put("loop_line_info",new BasicDBObject('$exists',false))
|
||||
// q.put("name","经纬天地园二期")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||
def keyMap = [:]
|
||||
for(def d : dbsour){
|
||||
keyMap.put(d.region+"##"+d.name, d)
|
||||
}
|
||||
return keyMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
//
|
||||
// DBObject q = new BasicDBObject();
|
||||
|
||||
+9
-3
@@ -73,6 +73,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
|
||||
def appendResult(def headers, def ret){
|
||||
|
||||
def sb = new StringBuffer();
|
||||
// sb.append(headers.join("@|@"))
|
||||
// sb.append(System.getProperty("line.separator"));
|
||||
@@ -121,11 +122,16 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def day = new Date().getDate()
|
||||
def startMDate = DateUtils.truncateDay(new Date()-1)
|
||||
def endMDate = DateUtils.truncateDay(new Date())
|
||||
if(day == 15){
|
||||
if(CommonDatautils.isWindow()){
|
||||
districts = getFzDistricts(city, null, null)
|
||||
}else{
|
||||
districts = getFzDistricts(city, startMDate, endMDate)
|
||||
} else{
|
||||
if(day == 15){
|
||||
districts = getFzDistricts(city, null, null)
|
||||
}else{
|
||||
districts = getFzDistricts(city, startMDate, endMDate)
|
||||
}
|
||||
}
|
||||
|
||||
def curPriceMap = findFzDistrictSellStat("福州",curDate)
|
||||
def result = []
|
||||
for(def rMap : districts){
|
||||
|
||||
+2
-1
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
@@ -63,7 +64,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def calcMonthlyPrice(){
|
||||
//只有福州使用tob价格
|
||||
//只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
|
||||
|
||||
|
||||
+7
-10
@@ -29,11 +29,11 @@ class FzDataResultStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
// task.priceMarkupRun();
|
||||
// task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
// task.testNewPrice()
|
||||
task.testNewPrice()
|
||||
}
|
||||
|
||||
def margeExcel(){
|
||||
@@ -82,7 +82,7 @@ class FzDataResultStat {
|
||||
}
|
||||
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
|
||||
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -4, "yyyyMM")))
|
||||
def map2 = [:]
|
||||
for(def obj : list2) {
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
@@ -119,7 +119,7 @@ class FzDataResultStat {
|
||||
index++
|
||||
println row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对5_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
@@ -301,16 +301,13 @@ class FzDataResultStat {
|
||||
}
|
||||
|
||||
def testNewPrice(){
|
||||
ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d->
|
||||
ExcelUtils.modify("D:\\数据\\福州\\福州200个小区抽样价格核验202507.xlsx",{d->
|
||||
if(d['A']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region",d['B'])
|
||||
q.put("name",d['C'])
|
||||
q.put("month", 202402)
|
||||
q.put("comm_id", d['A'])
|
||||
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
|
||||
if(obj){
|
||||
d['S'] = obj.avg_price
|
||||
d['D'] = obj.avg_price
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
+109
-1
@@ -13,11 +13,17 @@ import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.solr.common.util.DateUtil
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AddAvgpriceBatch {
|
||||
|
||||
private static def citys = null;
|
||||
private static def districtMap = [:];
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
|
||||
|
||||
static {
|
||||
citys = ["宜昌"]//CommonDatautils.findAllCity()
|
||||
@@ -26,10 +32,112 @@ class AddAvgpriceBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new AddAvgpriceBatch();
|
||||
task.run1();
|
||||
// task.run1();
|
||||
// task.test()
|
||||
// task.sendPriceCommand()
|
||||
// task.sendNewDistPrice()
|
||||
|
||||
// task.delNoExistsDistPrice()
|
||||
task.checkToBprice()
|
||||
}
|
||||
|
||||
def checkToBprice(){
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
for(def i=0;i<citys.size();i++){ //76
|
||||
def c = citys[i]
|
||||
def keyDistMap = CommonDatautils.findAllDistrictsKey(c, "district", ["city","region","name"])
|
||||
def rets = findNoPrices(c, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), keyDistMap)
|
||||
println c + " "+rets.size()
|
||||
addCommand(rets)
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addCommand(rets){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
def m = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
CommonUtils.recalculationAverage(m.city, m.district, m.name, m.estate_type, m.weekend,false,0)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findNoPrices(c, weekend, keyDistMap){
|
||||
def q = new BasicDBObject("city", c).append("weekend", new BasicDBObject('$gte', weekend)).append("avg_price_2b", null)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("district_name", 1)
|
||||
keys.put("estate_type", 1)
|
||||
keys.put("avg_price_2b", 1)
|
||||
keys.put("weekend", 1)
|
||||
def dbcursor = DIST_STAT_COLL.find(q, keys).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
def distObj = keyDistMap.get(d.region+"##"+d.district_name)
|
||||
if(distObj){
|
||||
if(!d.avg_price_2b) {
|
||||
rets << ["city": d.city, "district": d.region, "name": d.district_name,"estate_type":d.estate_type, "weekend": d.weekend]
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def delNoExistsDistPrice(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index<250){
|
||||
continue
|
||||
}
|
||||
println index +" "+c
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name"])
|
||||
def nameKeys = []
|
||||
for(def dist : dists){
|
||||
nameKeys << dist.region+"#"+dist.name
|
||||
}
|
||||
delDistPrice(c, nameKeys)
|
||||
}
|
||||
}
|
||||
|
||||
def delDistPrice(city, nameKeys){
|
||||
def q = new BasicDBObject("city", city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("district_name", 1)
|
||||
def dbcursor = DIST_STAT_COLL.find(q, keys).batchSize(5000)
|
||||
for(def d : dbcursor){
|
||||
def key = d.region +"#"+d.district_name
|
||||
if(!nameKeys.contains(key)){
|
||||
DIST_STAT_COLL.remove(new BasicDBObject("_id", d._id))
|
||||
println "删除:"+city +" "+ d.district_name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def sendNewDistPrice(){
|
||||
|
||||
@@ -1,21 +1,225 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.poi.xwpf.usermodel.XWPFDocument
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.Semaphore
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class DistrictStat {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictStat();
|
||||
|
||||
// task.run()
|
||||
task.run("无锡")
|
||||
// task.run("无锡")
|
||||
|
||||
// 合肥市
|
||||
// 芜湖市
|
||||
// 安庆市
|
||||
// 马鞍山市
|
||||
// 宣称市
|
||||
// 六安市
|
||||
// def citys = ["宣城","合肥","芜湖","安庆","马鞍山","六安"]
|
||||
// for(def city : citys){
|
||||
// task.stat(city)
|
||||
// }
|
||||
|
||||
|
||||
// task.cityStat()
|
||||
task.cityStat1()
|
||||
|
||||
}
|
||||
|
||||
def cityStat1(){
|
||||
def index = 0
|
||||
def rets = []
|
||||
ExcelUtils.modify("D:\\数据\\小区\\城市数量统计1.xlsx",{d->
|
||||
|
||||
def districts = null
|
||||
if(d['D']=="义乌"){
|
||||
districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"])
|
||||
def newDists = []
|
||||
for(def dist : districts){
|
||||
if(dist.region =="义乌"){
|
||||
newDists << dist
|
||||
}
|
||||
}
|
||||
districts = newDists
|
||||
}else if(d['D']=="金华"){
|
||||
districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"])
|
||||
def newDists = []
|
||||
for(def dist : districts){
|
||||
if(dist.region !="义乌"){
|
||||
newDists << dist
|
||||
}
|
||||
}
|
||||
districts = newDists
|
||||
}else if(d['D']=="无锡"){
|
||||
districts = CommonDatautils.findAllDistrictByCity("无锡", "district", ["region","name"])
|
||||
def newDists = []
|
||||
for(def dist : districts){
|
||||
if(dist.region !="江阴"){
|
||||
newDists << dist
|
||||
}
|
||||
}
|
||||
districts = newDists
|
||||
}else if(d['D']=="苏州"){
|
||||
districts = CommonDatautils.findAllDistrictByCity("苏州", "district", ["region","name"])
|
||||
def newDists = []
|
||||
for(def dist : districts){
|
||||
if(dist.region !="常熟"){
|
||||
newDists << dist
|
||||
}
|
||||
}
|
||||
districts = newDists
|
||||
}else if(d['D']=="济南"){
|
||||
districts = CommonDatautils.findAllDistrictByCity("济南", "district", ["region","name"])
|
||||
def newDists = []
|
||||
for(def dist : districts){
|
||||
if(dist.region !="莱芜"){
|
||||
newDists << dist
|
||||
}
|
||||
}
|
||||
districts = newDists
|
||||
}else{
|
||||
districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["region","name"])
|
||||
}
|
||||
|
||||
d['F'] = districts.size()
|
||||
if(districts){
|
||||
d['E'] = "是"
|
||||
}else{
|
||||
d['E'] = "否"
|
||||
}
|
||||
|
||||
def prov = null
|
||||
if(districts){
|
||||
index ++
|
||||
def data = fullCityMap.get(d['D'])
|
||||
if(data){
|
||||
prov = data.province
|
||||
}
|
||||
def str = districts.collect { it.name }.join(", ")
|
||||
rets << ["id":index, "type":"A", "province":prov, "city":d['D'], "names":str]
|
||||
|
||||
createDoc(rets)
|
||||
}
|
||||
}, 0)
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表1.dat", [], "utf-8", false);
|
||||
}
|
||||
|
||||
def createDoc(dataList){
|
||||
// 创建 Word 文档
|
||||
def doc = new XWPFDocument()
|
||||
def table = doc.createTable(dataList.size() + 1, 5) // 行数=数据行+表头,列数=3
|
||||
|
||||
// 设置表头
|
||||
def headerRow = table.getRow(0)
|
||||
|
||||
headerRow.getCell(0).setText("序号")
|
||||
headerRow.getCell(1).setText("类型")
|
||||
headerRow.getCell(2).setText("省份")
|
||||
headerRow.getCell(3).setText("城市")
|
||||
headerRow.getCell(4).setText("楼盘名称")
|
||||
|
||||
// 填充数据
|
||||
dataList.eachWithIndex { item, index ->
|
||||
def row = table.getRow(index+1) // 跳过表头
|
||||
row.getCell(0).setText(item.id.toString())
|
||||
row.getCell(1).setText(item.type.toString())
|
||||
row.getCell(2).setText(item.province.toString())
|
||||
row.getCell(3).setText(item.city.toString())
|
||||
row.getCell(4).setText(item.names.toString())
|
||||
}
|
||||
|
||||
// 保存到文件
|
||||
def outputFile = new File("A类城市楼盘列表.docx")
|
||||
doc.write(new FileOutputStream(outputFile))
|
||||
println "Word 表格生成成功: ${outputFile.absolutePath}"
|
||||
}
|
||||
|
||||
def cityStat(){
|
||||
|
||||
def index = 0
|
||||
def rets = []
|
||||
ExcelUtils.modify("D:\\数据\\小区\\城市数量统计.xlsx",{d->
|
||||
def districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["city", "region", "name"])
|
||||
if(districts){
|
||||
d['F'] = districts.size()
|
||||
d['E'] = "是"
|
||||
|
||||
rets << ["序号","类型","省份","城市","楼盘名称","金华"]
|
||||
def data = fullCityMap.get(d['D'])
|
||||
def prov = null
|
||||
if(data){
|
||||
prov = data.province
|
||||
}
|
||||
|
||||
for(def dist : districts){
|
||||
index++
|
||||
rets << [index, "A", prov,dist.city, dist.name, dist.region]
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\小区\\A类城市楼盘列表.xlsx", rets, "楼盘数据详情列表")
|
||||
FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表.dat", [], "utf-8", false);
|
||||
}
|
||||
|
||||
def stat(city){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name","alias"])
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
def countResult = Collections.synchronizedList(new ArrayList());
|
||||
def rets = []
|
||||
// def houseTotal = CommonDatautils.findHouseCountByDate(city, null, null, null, "sell", 20240611)
|
||||
rets << city
|
||||
rets << districts.size()
|
||||
// rets << houseTotal
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def o = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def count = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20240611)
|
||||
if(count){
|
||||
result << o
|
||||
countResult << count
|
||||
}
|
||||
println totalCount.get() +" "+o.region +" "+o.name+" "+count
|
||||
} catch (Exception e) {
|
||||
println totalCount.get() +" 查房源Error: "+city+" "+ o.region+" "+o.name+" "+e.getMessage()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
rets << countResult.sum()
|
||||
rets << result.size()
|
||||
|
||||
println rets
|
||||
}
|
||||
def run(def city){
|
||||
def weekend = DateUtils.parseFromTightYYYYMMDD("2020-10-25")
|
||||
|
||||
+4
-2
@@ -64,7 +64,9 @@ class FangjiatongUserStat {
|
||||
|
||||
def stat(){
|
||||
def users = findUses(12)
|
||||
def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
|
||||
// def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
|
||||
// def months = ["2020-01-01","2020-02-01","2020-03-01","2020-04-01","2020-05-01","2020-06-01","2020-07-01","2020-08-01","2020-09-01","2020-10-01","2020-11-01","2020-12-01","2021-01-01"]
|
||||
def months = ["2022-01-01","2022-02-01","2022-03-01","2022-04-01","2022-05-01","2022-06-01","2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01"]
|
||||
def rets = []
|
||||
def header = ["用户id","支行名称", "备注名称"]
|
||||
months.each{m->
|
||||
@@ -87,7 +89,7 @@ class FangjiatongUserStat {
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019")
|
||||
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2022")
|
||||
}
|
||||
|
||||
def findUses(role){
|
||||
|
||||
+20
-1
@@ -27,10 +27,29 @@ class SuzouPocTest {
|
||||
static main(args) {
|
||||
def task = new SuzouPocTest()
|
||||
// task.run()
|
||||
task.newRun()
|
||||
// task.newRun()
|
||||
task.test()
|
||||
}
|
||||
|
||||
def test(){
|
||||
ExcelUtils.modify("D:\\数据\\上海联合产权交易所\\第1批拟处置房产清单(长城大厦、台山碧桂园).xlsx", { d ->
|
||||
if(d['O']){
|
||||
def floorAdds = d['I'].split("/")
|
||||
def ret1 = OpenApiClient.addressEvaluation(d['O'], null, d['D'], d['D'], d['K'], floorAdds[0], floorAdds[1])
|
||||
if (ret1) {
|
||||
d['N'] = ret1.totalPrice
|
||||
d['M'] = ret1.avgPrice
|
||||
d['P'] = ret1.district
|
||||
d['Q'] = ret1.name
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def newRun(){
|
||||
|
||||
println("start:"+new Date().toString())
|
||||
def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls"
|
||||
newMatch(filePath)
|
||||
|
||||
@@ -21,7 +21,7 @@ class FileUtils {
|
||||
def sb = new StringBuffer();
|
||||
|
||||
if(headers){
|
||||
sb.append(headers.join("|@|"))
|
||||
sb.append(headers.join("@"))
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ class FileUtils {
|
||||
for(def row : ret){
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(StringUtils.join(row.toArray(), "@"))
|
||||
sb.append("\n");
|
||||
|
||||
if(index%10000==0){
|
||||
|
||||
+583
@@ -0,0 +1,583 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.newDistrict.InvokeHttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Element
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
class SzGovHouseCrawl {
|
||||
|
||||
static main(args) {
|
||||
def task = new SzGovHouseCrawl();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def index_content = """
|
||||
|
||||
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<title>一手房源公示</title>
|
||||
<link href="css/marster.css" rel="stylesheet" type="text/css" />
|
||||
</head>
|
||||
<body>
|
||||
<form name="Form1" method="post" action="index.aspx" id="Form1">
|
||||
<div>
|
||||
<input type="hidden" name="__EVENTTARGET" id="__EVENTTARGET" value="" />
|
||||
<input type="hidden" name="__EVENTARGUMENT" id="__EVENTARGUMENT" value="" />
|
||||
<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="nFQUad+408Aq5zCXFL9mwJbwfbUjHqOTwTmTIzIV/nn2wuuQIXExg/fWXnK8Ll6o8tHsGFJtGbKc87jIO8eGWmN56r/iP2F/Oi3sPuUM14rN/OFTR/y1yS4eQXiTUniZCpvc+hwWUXM6La+cm63hbAskQFo2Geb/8JeX/kCREa/vJqSoFTxJFuxoDevNRV5QeytYQ8PxV89z/L/KAGHwo7sR66wzuWciXwvLcXa6BWaA9aU953Rr1QnFrqy3QrAnmLmxfEtEbvw6cySgECeARwElfHdG0dKGni1inFrFf3lYDdNAL2gLoQoNFBIbDj/vXaYZQoAypynsqc8hiZotSa5aqehFlDxDu3TO3jRcWKQDue55UoGZURl8WyjlfLpebiAdSQirRRrLRQHcb4Baf7U0y/JiFFk0A75I9bfJM72eq16jjjhV9++DNXJRUn5KGMSn5h+PxwcsS8rSga0KRwsM9LDylCsNkedVucDhMCGrNwmbhAsnUIiMD7Va8wQizQllRtDJWPFZRU+YPUWxqRkfJnTEzdl8MQw7eEC6rweywfrDn3VLvw8RkRqAdaSyA5K3z4VXR0QVuBcX6teex3UIQQep9oWVvK6FGlpHEfkqEPYdwnyoEoGgRJHk/R0xhCx2ahrlYI04dbT2GaugM2JDVBnGWhQBYeylLukCVpBnaxUgiGG1Dx6lUBbLi0yFt3GkdYzi5XeTzQoGsMTbdVMBvcIqqFOs9dZBfebrWrpwTnAgUldkrtyw6j9Lf0sKPCUifvCCg8Ks1OnniSkpI1zJ0sfseGrnPQv7357IEg6UWyfvMFr3jjzqcX0QV35ajJMpzfaKIYGlzzIkQwEfjhNU83+nhI3yct8nyfmE2lROKDzOdCYFmlOPVQ1yF0vdPWs2cmx3SeQl53r1nPokHRQFuYKif4B7Tx/GN/1jPoKtrqBekL/8+ewdF1YAL8z7okg4LQ7DWaITUPvZmP7p/kAqilgKVMzqAOmdl5FzK1RwDn1fh6Ciw79aM3MPYwzkfxwA9S9uo67RuF4DHbVkuob/SJ1JGfXvFNJEvm/45UJWBFKEYWnmJJLvbeZSSZ455ox0aNLtzsTpTm3yr7S+LVNqTQIxEdFMed1gC/9caV753lkuEAHlxWwZHE3eULOw04aM8w==" />
|
||||
</div>
|
||||
|
||||
<script type="text/javascript">
|
||||
//<![CDATA[
|
||||
var theForm = document.forms['Form1'];
|
||||
if (!theForm) {
|
||||
theForm = document.Form1;
|
||||
}
|
||||
function __doPostBack(eventTarget, eventArgument) {
|
||||
if (!theForm.onsubmit || (theForm.onsubmit() != false)) {
|
||||
theForm.__EVENTTARGET.value = eventTarget;
|
||||
theForm.__EVENTARGUMENT.value = eventArgument;
|
||||
theForm.submit();
|
||||
}
|
||||
}
|
||||
//]]>
|
||||
</script>
|
||||
|
||||
|
||||
<script src="/ris/WebResource.axd?d=oAb-PGCUASjSAEHmsWLcW3nRSbp0kTk2wHZPA_xOoR1TX097F_c8lWeB0anDU_zTJ7nGX_3dSaDy8zzgj1ZXo1L1Sc41&t=638314433260000000" type="text/javascript"></script>
|
||||
|
||||
|
||||
<script src="/ris/ScriptResource.axd?d=ez7z8Rk-sRUjmvQcOq01fhxlJ1QkOJwcsZyID_Ni3ynenZjRnecb_-APpFxKSgOMhcW8F7rusL02bjz8mZvC6mdZ5fFUnv7hA8P09LlmyTRz8AzNNpmFk65QF_tFzWpkOb3lJg2&t=636978664134464033" type="text/javascript"></script>
|
||||
<script src="/ris/ScriptResource.axd?d=3ACyCjbvPx1mr6XvFjwUpUrueEjvCnwM6jay2gUKFWC2bAv9PJDw7_w8e-FijIouXVInB0UX5vw0I3Fbx5YA92gSlMgpLLv3eu8tE_KjYAT1gVhsDve7JfQShCjQ8wBnrY5v-Q2&t=636978664134464033" type="text/javascript"></script>
|
||||
<div>
|
||||
|
||||
\t<input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="2A35A6B2" />
|
||||
\t<input type="hidden" name="__VIEWSTATEENCRYPTED" id="__VIEWSTATEENCRYPTED" value="" />
|
||||
\t<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="c/yjhDZvdy6HmhcL4EkG1J1Z3q6SMKFIycFC2eMEcd9PHzbGDxG56wrvw1REP793VkDqjnfrk1++ArpjUL0ql/yAB3MB5SUwn7h0YzAXw3/YG2ipVYdaOarImjt0o8JYuszLTaNRVxXmK8KTgUXOhpL0mroOqigxEaXUaeSKml9LBl2xzJttjM+uErnLyCLXSyLW0qjXWs+pCPqD5SgIhBUuVmg=" />
|
||||
</div>
|
||||
|
||||
<div class="wrap">
|
||||
<div class="record fix mb10">
|
||||
|
||||
<script type="text/javascript">
|
||||
//<![CDATA[
|
||||
Sys.WebForms.PageRequestManager._initialize('scriptManager2', document.getElementById('Form1'));
|
||||
Sys.WebForms.PageRequestManager.getInstance()._updateControls(['tupdatepanel2'], [], [], 90);
|
||||
//]]>
|
||||
</script>
|
||||
|
||||
|
||||
<div class="recordLinkbox recordLink">
|
||||
所在区:<a id="hypAll" href="javascript:__doPostBack('hypAll','')" style="color:Red;">全市</a>
|
||||
<a id="hypBa" href="javascript:__doPostBack('hypBa','')" style="color:Black;">宝安</a>
|
||||
<a id="hypFt" href="javascript:__doPostBack('hypFt','')" style="color:Black;">福田</a>
|
||||
<a id="hypLg" href="javascript:__doPostBack('hypLg','')" style="color:Black;">龙岗</a>
|
||||
<a id="hypLh" href="javascript:__doPostBack('hypLh','')" style="color:Black;">罗湖</a>
|
||||
<a id="hypNs" href="javascript:__doPostBack('hypNs','')" style="color:Black;">南山</a>
|
||||
<a id="hypQh" href="javascript:__doPostBack('hypQh','')" style="color:Black;">前海</a>
|
||||
<a id="hypYt" href="javascript:__doPostBack('hypYt','')" style="color:Black;">盐田</a>
|
||||
<a id="hypLhQ" href="javascript:__doPostBack('hypLhQ','')" style="color:Black;">龙华</a>
|
||||
<a id="hypGm" href="javascript:__doPostBack('hypGm','')" style="color:Black;">光明</a>
|
||||
<a id="hypPs" href="javascript:__doPostBack('hypPs','')" style="color:Black;">坪山</a>
|
||||
<a id="hypDp" href="javascript:__doPostBack('hypDp','')" style="color:Black;">大鹏</a>
|
||||
<a id="hypSSHZ" href="javascript:__doPostBack('hypSSHZ','')">深汕合作</a></div>
|
||||
<div class="deveBox" style="line-height: 0px;">
|
||||
<p class="left mr25">
|
||||
项目查询 <input name="tep_name" type="text" id="tep_name" style="height:28px;width:150px;" /></p>
|
||||
<p class="left mr25">
|
||||
开发企业 <input name="organ_name" type="text" value="深圳市城建产业园发展有限公司" id="organ_name" style="height:28px;width:150px;" /></p>
|
||||
<p class="left mr25">
|
||||
宗地地址 <input name="site_address" type="text" id="site_address" style="height:28px;width:150px;" /></p>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<input type="submit" name="Button1" value="查询" id="Button1" style="background: #1956c6;
|
||||
width: 75px; height: 28px; line-height: 28px; display: block; text-align: center; cursor:pointer;
|
||||
font-size: 14px; color: #fff;border-radius: 2px;" />
|
||||
|
||||
</div>
|
||||
<div id="updatepanel2">
|
||||
\t
|
||||
<div class="record fix">
|
||||
<div class="fix">
|
||||
|
||||
<table class="table ta-c bor-b-1 table-white">
|
||||
<tr>
|
||||
<th style="width: 92px;">
|
||||
序号
|
||||
</th>
|
||||
<th style="width: 220px;">
|
||||
预售证号
|
||||
</th>
|
||||
<th style="width: 312px;">
|
||||
项目名称
|
||||
</th>
|
||||
<th style="width: 270px;">
|
||||
开发企业
|
||||
</th>
|
||||
<th style="width: 110px;">
|
||||
所在区
|
||||
</th>
|
||||
<th style="width: 132px;">
|
||||
批准时间
|
||||
</th>
|
||||
</tr>
|
||||
|
||||
<tr class="">
|
||||
<td>
|
||||
1
|
||||
</td>
|
||||
<td>
|
||||
<a href='./certdetail.aspx?id=133423' target='_self'>
|
||||
深房许字(2024)罗湖002号</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href='projectdetail.aspx?id=133423' target='_self'>
|
||||
凌云华府</a>
|
||||
</td>
|
||||
<td>
|
||||
深圳市城建产业园发展有限公司
|
||||
</td>
|
||||
<td>
|
||||
罗湖
|
||||
</td>
|
||||
<td>
|
||||
2024-06-27
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr class="bg-grayf7">
|
||||
<td>
|
||||
2
|
||||
</td>
|
||||
<td>
|
||||
<a href='./certdetail.aspx?id=124226' target='_self'>
|
||||
深房许字(2022)罗湖007号</a>
|
||||
</td>
|
||||
<td>
|
||||
<a href='projectdetail.aspx?id=124226' target='_self'>
|
||||
高投大厦</a>
|
||||
</td>
|
||||
<td>
|
||||
深圳市城建产业园发展有限公司
|
||||
</td>
|
||||
<td>
|
||||
罗湖
|
||||
</td>
|
||||
<td>
|
||||
2022-12-16
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<tr style="height:18px;">
|
||||
<td colspan="6">
|
||||
|
||||
</td>
|
||||
</tr>
|
||||
</tr></table>
|
||||
</div>
|
||||
<div class="page">
|
||||
<div class="pages pagehtml right">
|
||||
|
||||
\t<!-- AspNetPager V7.0.2 for VS2005 & VS2008 Copyright:2003-2007 Webdiyer (www.webdiyer.com) -->
|
||||
\t<!--记录总数只有一页,AspNetPager已自动隐藏,若需在只有一页数据时显示AspNetPager,请将AlwaysShow属性值设为true!-->
|
||||
\t<!-- AspNetPager V7.0.2 for VS2005 & VS2008 End -->
|
||||
|
||||
\t
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<script type="text/javascript">
|
||||
//<![CDATA[
|
||||
Sys.Application.initialize();
|
||||
//]]>
|
||||
</script>
|
||||
</form>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
"""
|
||||
|
||||
getRoomRefDetail("https://zjj.sz.gov.cn/ris/bol/szfdc/housedetail.aspx?id=1979350",[:])
|
||||
def doc = Jsoup.parse(index_content);
|
||||
Elements trs = doc.select("table tr")
|
||||
def name = "深圳市城建产业园发展有限公司"
|
||||
def results = []
|
||||
def header = ["楼层","房间号","单元号","销售状态","楼栋名称","预售许可证号","项目名称","所属项目地址","开发商名称","开盘日期","监管银行","监管账户名称","监管账户账号","房屋用途","户型","预测总建筑面积","预测套内建筑面积","预测公摊建筑面积","实际总建筑面积","实际套内建筑面积","实际公摊建筑面积","城市","拟售价格-按建筑面积计","拟售价格-按套内建筑面积计","预售总套数","现售总套数","栋数"]
|
||||
results << header
|
||||
//开发商查询结果页
|
||||
println trs.size()-2 +" 预售证"
|
||||
for(def i=1 ;i<trs.size();i++){
|
||||
def href1 = trs[i].select("td:eq(1) a").attr("href")
|
||||
if(href1){
|
||||
href1 = "https://zjj.sz.gov.cn/ris/bol/szfdc"+ href1.substring(1)
|
||||
def href2 = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+trs[i].select("td:eq(2) a").attr("href")
|
||||
|
||||
def sellObjMap = getSellCodeObject(href1) //预售证页面
|
||||
println i+ " 预售证页面 加载完成"
|
||||
def projectMap = getProjectDetail(href2) //抓取项目页面
|
||||
println i+ " 预售证页面 抓取项目页面"
|
||||
if(projectMap){
|
||||
def projList = projectMap.projList
|
||||
for(def pro : projList){
|
||||
def nowsellrooms = pro.yusell_roomDetails
|
||||
for(def yusell : nowsellrooms){
|
||||
def row = []
|
||||
row << yusell["楼层"]
|
||||
row << yusell["房号"]
|
||||
row << yusell["座号"]
|
||||
row << yusell["房号状态"]
|
||||
row << pro["楼名"]
|
||||
row << sellObjMap['许可证号']
|
||||
row << sellObjMap['项目名称']
|
||||
row << sellObjMap['所在位置']
|
||||
row << sellObjMap['发展商']
|
||||
row << sellObjMap['发证日期 (开始销售日期)']
|
||||
row << projectMap['价款监管机构1']
|
||||
row << projectMap['账户名称']
|
||||
row << projectMap['账号']
|
||||
row << yusell["用途"]
|
||||
row << ""
|
||||
row << yusell["预售-建筑面积"]
|
||||
row << yusell["预售-套内建筑面积"]
|
||||
row << yusell["预售-分摊面积"]
|
||||
row << yusell["竣工-建筑面积"]
|
||||
row << yusell["竣工-套内建筑面积"]
|
||||
row << yusell["竣工-分摊面积"]
|
||||
row << "深圳"
|
||||
row << yusell["拟售价格-按建筑面积计"]
|
||||
row << yusell["拟售价格-按套内建筑面积计"]
|
||||
row << projectMap['预售总套数']
|
||||
row << projectMap['现售总套数']
|
||||
row << sellObjMap["栋数"]
|
||||
results << row
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\新房\\${name}.xlsx", results, "深圳")
|
||||
}
|
||||
|
||||
def getSellCodeObject(url){
|
||||
String content = new HttpRequest().get(url, null, "utf-8")
|
||||
def doc = Jsoup.parse(content);
|
||||
Elements trs = doc.select("table tr")
|
||||
def sellObjMap = [:]
|
||||
def sellUseList = []
|
||||
for(def i=1 ;i<trs.size();i++){
|
||||
def tds = trs[i].select("td")
|
||||
if(tds.size()==4){
|
||||
sellObjMap.put(tds[0].text(), tds[1].text())
|
||||
sellObjMap.put(tds[2].text(), tds[3].text())
|
||||
}else if(tds.size()==6){
|
||||
if(tds[1].text()!="--"){
|
||||
def map = [:]
|
||||
map.put(tds[0].text(), tds[1].text())
|
||||
map.put(tds[2].text(), tds[3].text())
|
||||
map.put(tds[4].text(), tds[5].text())
|
||||
sellUseList << map
|
||||
}
|
||||
}else if(tds.size()==2){
|
||||
sellObjMap.put(tds[0].text(), tds[1].text())
|
||||
}
|
||||
}
|
||||
sellObjMap.useList = sellUseList
|
||||
return sellObjMap;
|
||||
}
|
||||
|
||||
def getProjectDetail(url){
|
||||
String content = new HttpRequest().get(url, null, "utf-8")
|
||||
def doc = Jsoup.parse(content);
|
||||
// def sellInfoUrl = doc.select("strong:contains(卖方信息) a").attr("href")
|
||||
// def devMap = getSellInfo("https://zjj.sz.gov.cn/ris/bol/szfdc/"+sellInfoUrl)
|
||||
|
||||
Elements tableEle = doc.select("table")
|
||||
Elements trs1 = tableEle.get(0).select("tr")
|
||||
def projMap = [:]
|
||||
|
||||
for(def i=1 ;i<trs1.size();i++) {
|
||||
def tds = trs1[i].select("td")
|
||||
|
||||
for (def j = 0; j+2 <= tds.size(); j=j+2) {
|
||||
projMap.put(tds[j].text(), tds[j + 1].text())
|
||||
}
|
||||
}
|
||||
|
||||
Elements trs2 = tableEle.get(1).select("tr")
|
||||
def projList = []
|
||||
for(def i=1 ;i<trs2.size();i++) {
|
||||
def tds = trs2[i].select("td")
|
||||
def map = [:]
|
||||
map.put("项目名称", tds[0].text())
|
||||
map.put("楼名", tds[1].text())
|
||||
map.put("建设工程规划许可证", tds[2].text())
|
||||
map.put("建筑工程施工许可证", tds[3].text())
|
||||
def roomUrl = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+tds[4].select("a").attr("href").replace("&","&")
|
||||
map.put("套房信息", roomUrl)
|
||||
|
||||
|
||||
def yusellroomList = getYuSellRoomDetail(roomUrl)
|
||||
map.yusell_roomDetails = yusellroomList
|
||||
|
||||
// def nowsellroomList = getNowSellRoomDetail(roomUrl)
|
||||
// map.nowsellroomDetails = nowsellroomList
|
||||
projList << map
|
||||
}
|
||||
projMap.projList = projList
|
||||
// projMap.devInfoMap = devMap
|
||||
return projMap
|
||||
}
|
||||
|
||||
def getYuSellRoomDetail(roomUrl){
|
||||
String content = new HttpRequest().get(roomUrl, null, "utf-8")
|
||||
def doc = Jsoup.parse(content);
|
||||
def roomList = []
|
||||
// Elements tdsEle = doc.select("td:contains(房号:)")
|
||||
|
||||
// for(def td : tdsEle){
|
||||
// def tdText = td.text()
|
||||
// if(tdText.contains("房号")){
|
||||
// def roomMap = [:]
|
||||
// def divEle = td.select("div")
|
||||
// roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
|
||||
// roomMap.put("房号状态", divEle.get(1).text())
|
||||
// def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+divEle.get(1).select("a").attr("href")
|
||||
// roomMap.put("房号url", href)
|
||||
// roomMap = getRoomRefDetail(href, roomMap)
|
||||
// roomList << roomMap
|
||||
// }
|
||||
// }
|
||||
|
||||
def hrefs = doc.select("#divShowBranch a")
|
||||
for(def hrefObj : hrefs){
|
||||
def danyuan = hrefObj.text().replace("[", "").replace("]", "")
|
||||
def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/" + hrefObj.attr("href")
|
||||
def danyuan_list = getDanyuanPage(href)
|
||||
roomList.addAll(danyuan_list)
|
||||
}
|
||||
return roomList
|
||||
}
|
||||
|
||||
def getDanyuanPage(roomUrl){
|
||||
String content = new HttpRequest().get(roomUrl, null, "utf-8")
|
||||
def doc = Jsoup.parse(content);
|
||||
|
||||
Elements tdsEle = doc.select("td:contains(房号:)")
|
||||
def roomList = []
|
||||
for(def td : tdsEle){
|
||||
def tdText = td.text()
|
||||
if(tdText.contains("房号")){
|
||||
def roomMap = [:]
|
||||
def divEle = td.select("div")
|
||||
roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
|
||||
roomMap.put("房号状态", divEle.get(1).text())
|
||||
def href = divEle.get(1).select("a").attr("href")
|
||||
|
||||
if(href){
|
||||
href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+ href
|
||||
roomMap.put("房号url", href)
|
||||
roomMap = getRoomRefDetail(href, roomMap)
|
||||
}
|
||||
roomList << roomMap
|
||||
}
|
||||
}
|
||||
return roomList
|
||||
}
|
||||
|
||||
def getContent(href){
|
||||
String content = null
|
||||
def error = 0
|
||||
while (error<=5){
|
||||
try{
|
||||
content = new HttpRequest().get(href, null, "utf-8")
|
||||
if(content){
|
||||
error = 100
|
||||
}else{
|
||||
Thread.sleep(3000)
|
||||
error ++
|
||||
}
|
||||
}catch(Exception e){
|
||||
println e.getMessage() + " "+ href
|
||||
Thread.sleep(3000)
|
||||
error ++
|
||||
}
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
def getRoomRefDetail(href, infoMap){
|
||||
String content = getContent(href)
|
||||
def doc = Jsoup.parse(content);
|
||||
Elements trsEle = doc.select("table tr")
|
||||
println href +" "+trsEle.size()
|
||||
if(trsEle.size()==0){
|
||||
def error = 0
|
||||
while (error<=3){
|
||||
Thread.sleep(3000)
|
||||
content = new HttpRequest().get(href, null, "utf-8")
|
||||
doc = Jsoup.parse(content);
|
||||
trsEle = doc.select("table tr")
|
||||
println href +" "+trsEle.size()
|
||||
if(trsEle.size()>0){
|
||||
error = 100
|
||||
}else{
|
||||
error ++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
for(def i=0;i<=1;i++){
|
||||
def tds = trsEle[i].select("td")
|
||||
for (def j = 0; j+2 <= tds.size(); j=j+2) {
|
||||
infoMap.put(tds[j].text(), tds[j + 1].text())
|
||||
}
|
||||
}
|
||||
|
||||
def tds1 = trsEle.get(2).select("td")
|
||||
infoMap.put("拟售价格-按建筑面积计", tds1[1].text().replace("--元/平方米","").replace("(按建筑面积计)",""))
|
||||
infoMap.put("拟售价格-按套内建筑面积计", tds1[2].text().replace("--元/平方米","").replace("(按套内建筑面积计)",""))
|
||||
infoMap.put("合同号", tds1[4].text())
|
||||
|
||||
|
||||
def tds2 = trsEle.get(4).select("td")
|
||||
for (def j = 0; j+2 <= tds2.size(); j=j+2) {
|
||||
infoMap.put("预售-"+tds2[j].text(), tds2[j + 1].text().replace("平方米",""))
|
||||
}
|
||||
|
||||
def tds3 = trsEle.get(6).select("td")
|
||||
for (def j = 0; j+2 <= tds3.size(); j=j+2) {
|
||||
infoMap.put("竣工-"+tds3[j].text(), tds3[j + 1].text().replace("平方米",""))
|
||||
}
|
||||
return infoMap;
|
||||
}
|
||||
|
||||
def getSellInfo(url){
|
||||
String content = new HttpRequest().get(url, null, "utf-8")
|
||||
def doc = Jsoup.parse(content);
|
||||
Elements trs1 = doc.select("table tr")
|
||||
def devMap = [:]
|
||||
for(def i=1 ;i<trs1.size();i++) {
|
||||
def tds = trs1[i].select("td")
|
||||
for (def j = 0; j+2 <= tds.size(); j=j+2) {
|
||||
devMap.put(tds[j].text(), tds[j + 1].text())
|
||||
}
|
||||
}
|
||||
return devMap
|
||||
}
|
||||
|
||||
|
||||
def getNowSellRoomDetail(url){
|
||||
// def doc = Jsoup.parse(content);
|
||||
// def header = [:]
|
||||
// def VIEWSTATE = doc.select("#__VIEWSTATE").attr("value")
|
||||
// def VIEWSTATEGENERATOR = doc.select("#__VIEWSTATEGENERATOR").attr("value")
|
||||
// def EVENTVALIDATION = doc.select("#__EVENTVALIDATION").attr("value")
|
||||
// doc.select("#__VIEWSTATEGENERATOR").attr("value")
|
||||
// header.put("__VIEWSTATE", doc.select("#__VIEWSTATE").attr("value"))
|
||||
// header.put("__EVENTTARGET", "imgBt2")
|
||||
// header.put("__VIEWSTATEGENERATOR", doc.select("#__VIEWSTATEGENERATOR").attr("value"))
|
||||
// header.put("__EVENTVALIDATION", doc.select("#__EVENTVALIDATION").attr("value"))
|
||||
//// params.put("scriptManager1","updatepanel1|imgBt2")
|
||||
//// String res = InvokeHttpRequest.post(url, params, "utf-8");
|
||||
// getRoomNowSellTabInfo(url, VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
|
||||
//
|
||||
// def params = [:]
|
||||
// params.put("id", "49636")
|
||||
// params.put("presellid", "129765")
|
||||
// HttpRequest re = new HttpRequest();
|
||||
// String res = re.post(url, params, header, "utf-8");
|
||||
|
||||
String res = new HttpRequest().get(url+"&Branch=未命名&isBlock=xs", null, "utf-8")
|
||||
def doc1 = Jsoup.parse(res);
|
||||
|
||||
Elements tdsEle = doc1.select("td:contains(房号:)")
|
||||
def roomList = []
|
||||
for(def td : tdsEle){
|
||||
def tdText = td.text()
|
||||
if(tdText.contains("房号")){
|
||||
def roomMap = [:]
|
||||
def divEle = td.select("div")
|
||||
roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
|
||||
roomMap.put("房号状态", divEle.get(1).text())
|
||||
def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+divEle.get(1).select("a").attr("href")
|
||||
roomMap.put("房号url", href)
|
||||
|
||||
roomMap = getRoomRefDetail(href, roomMap)
|
||||
roomList << roomMap
|
||||
}
|
||||
}
|
||||
return roomList
|
||||
}
|
||||
|
||||
|
||||
def getRoomNowSellTabInfo(targetUrl, VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION){
|
||||
try {
|
||||
// 创建连接
|
||||
URL url = new URL(targetUrl);
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
|
||||
// 基础配置
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setDoOutput(true); // 允许写入输出流
|
||||
conn.setConnectTimeout(5000); // 5秒连接超时
|
||||
conn.setReadTimeout(10000); // 10秒读取超时
|
||||
|
||||
// 设置请求头(模拟浏览器)
|
||||
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
|
||||
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
|
||||
conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9");
|
||||
|
||||
// 构建Form Data参数(带中文需编码)
|
||||
String formData = "__VIEWSTATE=" + URLEncoder.encode(VIEWSTATE, "UTF-8") +
|
||||
"&__VIEWSTATEGENERATOR=" + URLEncoder.encode(VIEWSTATEGENERATOR, "UTF-8") +
|
||||
"&__EVENTVALIDATION="+URLEncoder.encode(EVENTVALIDATION, "UTF-8") +
|
||||
"&__EVENTTARGET=imgBt2";
|
||||
|
||||
// 发送请求体
|
||||
OutputStream os = conn.getOutputStream()
|
||||
byte[] input = formData.getBytes("utf-8");
|
||||
os.write(input, 0, input.length);
|
||||
|
||||
|
||||
// 获取响应状态码
|
||||
int status = conn.getResponseCode();
|
||||
System.out.println("响应状态码: " + status);
|
||||
|
||||
String responseLine;
|
||||
// 读取响应内容
|
||||
if (status == HttpURLConnection.HTTP_OK) {
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"))
|
||||
StringBuilder response = new StringBuilder();
|
||||
|
||||
while ((responseLine = br.readLine()) != null) {
|
||||
response.append(responseLine.trim());
|
||||
}
|
||||
System.out.println("响应内容:\n" + response.toString());
|
||||
} else {
|
||||
System.out.println("请求失败,错误码: " + status);
|
||||
}
|
||||
return responseLine
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
+64
-2
@@ -3,10 +3,12 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang3.SerializationUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@@ -14,15 +16,30 @@ class AddrAlaisProcess {
|
||||
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
//
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
static main(args) {
|
||||
def task = new AddrAlaisProcess()
|
||||
|
||||
// task.run()
|
||||
// task.splitDist()
|
||||
// task.addAddr()
|
||||
task.nineApi()
|
||||
// task.nineApi()
|
||||
// task.createResult()
|
||||
// task.mergeResult()
|
||||
// task.addLocation()
|
||||
task.updateAddr()
|
||||
}
|
||||
|
||||
def addLocation(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\地图无路.xlsx",{d->
|
||||
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
if(obj){
|
||||
d['G'] = obj.lng/10000000000.0
|
||||
d['H'] = obj.lat/10000000000.0
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def splitDist(){
|
||||
@@ -102,7 +119,7 @@ class AddrAlaisProcess {
|
||||
def nineApi(){
|
||||
|
||||
def i = 0;
|
||||
while(i<20) {
|
||||
while(i<1) {
|
||||
def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx"
|
||||
@@ -122,6 +139,16 @@ class AddrAlaisProcess {
|
||||
def roadObj = findBaiduDatas( data.city, d['H'])
|
||||
if(roadObj){
|
||||
d['J'] = "百度存在"
|
||||
}else{
|
||||
// def distData = mcaDistMap.get(d['D'] + "#" + d['E'])
|
||||
// def fullDistName = null
|
||||
// if (distData) {
|
||||
// fullDistName = distData.district
|
||||
// }
|
||||
// def obj = CommonDatautils.findGaoDeDatas(data.city, fullDistName, d['H'])
|
||||
// if(obj){
|
||||
// d['J'] = "高德存在"
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -208,4 +235,39 @@ class AddrAlaisProcess {
|
||||
def obj = BAIDU_POI_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
|
||||
def updateAddr(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\地址及地址别名-详情-1.xlsx",{d->
|
||||
def obj = CommonDatautils.findDistrictById(d['C'])
|
||||
if(obj && d['G']){
|
||||
def flag = DataHandleUtils.isStandardRoad(d['L'])
|
||||
def udata = new BasicDBObject()
|
||||
if(obj.address == d['H']){
|
||||
udata.put("address", d['L'])
|
||||
}else{
|
||||
if(obj.addr_alias){
|
||||
def new_addr_alias = SerializationUtils.clone(obj.addr_alias)
|
||||
for(def alias : new_addr_alias){
|
||||
if(d['H'] == alias){
|
||||
if(flag){
|
||||
if(d['L']!=alias){
|
||||
alias = d['L']
|
||||
}
|
||||
}else{
|
||||
alias = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
if(new_addr_alias!=obj.addr_alias){
|
||||
udata.put("addr_alias", new_addr_alias)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', obj._id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+3
-5
@@ -225,15 +225,12 @@ class AutoAddOfficeBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def repeatDist(){
|
||||
def citys = ["南京"]//CommonDatautils.findAllCity().reverse()
|
||||
def citys = ["成都"]//CommonDatautils.findAllCity().reverse()
|
||||
def allrets = []
|
||||
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
|
||||
def index = 0
|
||||
for (def c : citys) {
|
||||
index++
|
||||
if(["宁波","无锡","青岛"].contains(c)){
|
||||
continue
|
||||
}
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
|
||||
def rets = checkRepeatDistrictBatch(c, districts)
|
||||
println c + " " + districts.size()
|
||||
@@ -254,13 +251,14 @@ class AutoAddOfficeBatch extends BaseBatch {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def cityId = cityCodeDataMap.get(city)
|
||||
|
||||
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502)
|
||||
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202505)
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def total = districts.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def sets = new CopyOnWriteArraySet()
|
||||
for (def i = 0; i < total; i++) {
|
||||
println i
|
||||
final def o1 = districts[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CommunityShapeProcess {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new CommunityShapeProcess();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", "河北省")
|
||||
q.put("shape", new BasicDBObject('$exists', true))
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("shape", 1)
|
||||
def dbcursor = community_CONN.find(q, keys).batchSize(1000)
|
||||
for(def d : dbcursor){
|
||||
if(d.shape.type=="Polygon"){
|
||||
def coordinates = d.shape.coordinates.get(0)
|
||||
def points = []
|
||||
def index = 0
|
||||
for(def arrs : coordinates){
|
||||
if(index!=0 && index+1!= coordinates.size()){
|
||||
if(!points.contains(arrs)){
|
||||
points << arrs
|
||||
}else{
|
||||
println d._id +" 重复数据:"+arrs
|
||||
}
|
||||
}else{
|
||||
points << arrs
|
||||
}
|
||||
index++
|
||||
}
|
||||
if(points.size()!=coordinates.size()){
|
||||
d.shape.coordinates = [points]
|
||||
def udata = new BasicDBObject()
|
||||
udata.shape = d.shape
|
||||
community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}else{
|
||||
println d._id
|
||||
def coordinateArrs = d.shape.coordinates
|
||||
def multiPoints = []
|
||||
def updateFlag = false
|
||||
for(def typeCoordinates : coordinateArrs){
|
||||
def pointArrs = []
|
||||
|
||||
for(def coordinates : typeCoordinates){
|
||||
def points = []
|
||||
def index = 0
|
||||
for(def arrs : coordinates){
|
||||
if(index!=0 && index+1!= coordinates.size()){
|
||||
if(!points.contains(arrs)){
|
||||
points << arrs
|
||||
}else{
|
||||
println d._id +" 重复数据:"+arrs
|
||||
updateFlag = true
|
||||
}
|
||||
}else{
|
||||
points << arrs
|
||||
}
|
||||
index++
|
||||
}
|
||||
pointArrs << points
|
||||
}
|
||||
multiPoints << pointArrs
|
||||
}
|
||||
|
||||
if(updateFlag){
|
||||
d.shape.coordinates = multiPoints
|
||||
def udata = new BasicDBObject()
|
||||
udata.shape = d.shape
|
||||
community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
+17
-6
@@ -93,12 +93,23 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def aiDevTown(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
|
||||
def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
|
||||
def message = res.data.replaceAll("\n", "")
|
||||
d['F'] = res.code
|
||||
d['G'] = message
|
||||
println d['E'] +" "+res.code + " "+message
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
|
||||
// def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
|
||||
// def message = res.data.replaceAll("\n", "")
|
||||
// d['F'] = res.code
|
||||
// d['G'] = message
|
||||
// println d['E'] +" "+res.code + " "+message
|
||||
def q = new BasicDBObject("name", d['E']).append("type","乡镇街道")
|
||||
def dbcursor = DEV_MAPPING_CONN.find(q)
|
||||
if(dbcursor.count()>0){
|
||||
def list = []
|
||||
for(def db : dbcursor){
|
||||
list << db.township
|
||||
}
|
||||
|
||||
d['H']= list.join(",")
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.looplineinfo.InPolygon
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.awt.geom.Point2D
|
||||
|
||||
class DistrictLoopInfo {
|
||||
|
||||
DBCollection LATLNG_POLYGON_COLL = MongoUtils.getCollectionByName("fangjia", "latlng_polygon", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLoopInfo()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", new BasicDBObject('$in', ["内环","中环","外环","郊环"]))
|
||||
def loopShapes = LATLNG_POLYGON_COLL.find(q)
|
||||
def loopMap = [:]
|
||||
for(def loop : loopShapes){
|
||||
loopMap.put(loop.name, loop)
|
||||
}
|
||||
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city", "region", "name", "lat", "lng", "loop_line_info"])
|
||||
println districts.size()
|
||||
def index = 0
|
||||
for(def d : districts){
|
||||
index++
|
||||
if(d.lng){
|
||||
def lng = d.lng/10000000000.0
|
||||
def lat = d.lat/10000000000.0
|
||||
Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6, lng), DataHandleUtils.getDoubleRound(6, lat));
|
||||
def loop1 = loopMap.get("内环")
|
||||
def flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
|
||||
def loop_line_info = ""
|
||||
if(flag){
|
||||
loop_line_info = "内环内"
|
||||
}else{
|
||||
loop1 = loopMap.get("中环")
|
||||
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
|
||||
if(flag){
|
||||
loop_line_info = "内中环"
|
||||
}else{
|
||||
loop1 = loopMap.get("外环")
|
||||
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
|
||||
if(flag){
|
||||
loop_line_info = "中外环"
|
||||
}else{
|
||||
loop1 = loopMap.get("郊环")
|
||||
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
|
||||
if(flag){
|
||||
loop_line_info = "外郊环"
|
||||
}else{
|
||||
loop_line_info = "郊环外"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println index +" "+d.region +" "+d.name+" 原:"+d.loop_line_info+" 现:"+loop_line_info+" lng:"+lng+","+lat
|
||||
def udata =new BasicDBObject()
|
||||
udata.put("loop_line_info", loop_line_info)
|
||||
DBUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set',udata))
|
||||
}else{
|
||||
println index + " "+d.region +" "+d.name+"无经纬度信息"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
-1
@@ -22,7 +22,47 @@ class DistrictRepeatBatch {
|
||||
static main(args) {
|
||||
def task = new DistrictRepeatBatch()
|
||||
// task.run()
|
||||
task.resultProcess()
|
||||
// task.resultProcess()
|
||||
task.processRepeatAndAlias()
|
||||
}
|
||||
|
||||
def processRepeatAndAlias(){
|
||||
ExcelUtils.modify("D:\\数据\\小区\\重复小区\\结果\\重复可疑小区-南京.xlsx",{d->
|
||||
if(d['H'] && d['H'] == "删除"){
|
||||
def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
if(!obj){
|
||||
obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
}
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
saveOptLog(obj.city, obj.region, obj.name, "remove")
|
||||
}
|
||||
}else{
|
||||
if(d['I']){
|
||||
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(obj){
|
||||
def arrs = d['I'].split(",")
|
||||
if(arrs){
|
||||
def alias = obj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
alias.addAll(arrs)
|
||||
alias = alias.unique()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("alias", alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
saveOptLog(obj.city, obj.region, obj.name, "update")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class GXDdistrictMatch {
|
||||
class 8GXDdistrictMatch {
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
|
||||
+137
-6
@@ -29,8 +29,8 @@ class TownMiddleProcess {
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping_0526", MONGO100_URL)
|
||||
DBCollection TEMP_ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
@@ -65,7 +65,7 @@ class TownMiddleProcess {
|
||||
//修改乡镇街道别名
|
||||
// updateTownAlias()
|
||||
//中间表同步到100
|
||||
synchTown()
|
||||
// synchTown()
|
||||
//添加地级市标志
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
@@ -101,11 +101,22 @@ class TownMiddleProcess {
|
||||
|
||||
// updateNewRegionPoyline()//更新新的区域围栏
|
||||
|
||||
updateMappingPolyline()
|
||||
// updateMappingPolyline()
|
||||
|
||||
//批量修改多边型围栏数据结构
|
||||
// updatePolylineBatch()
|
||||
// test2dshape()
|
||||
|
||||
//将省会城市更新到数据库中
|
||||
// updateProvinceCenter()
|
||||
//城市级别更新到库中
|
||||
// updateCityLevel()
|
||||
//城市顺序
|
||||
// updateCityNum()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createProvince(){
|
||||
def cat = "省"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
@@ -457,16 +468,17 @@ class TownMiddleProcess {
|
||||
def datas = []
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
if (datas.size() >= 2000) {
|
||||
if (datas.size() >= 5000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
// ADMINISTRATIVE_MAPPING_COLL.insert(datas)
|
||||
println datas.size()
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_COLL.insert(datas)
|
||||
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
datas.clear();
|
||||
}
|
||||
@@ -1449,4 +1461,123 @@ class TownMiddleProcess {
|
||||
}
|
||||
|
||||
|
||||
def updatePolylineBatch(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("shape",new BasicDBObject('$exists', true))
|
||||
q.put("update_shape",new BasicDBObject('$exists', false))
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100)
|
||||
for(def d : dbcursor){
|
||||
def type = d.shape.type
|
||||
def udata = new BasicDBObject()
|
||||
def shapeMap = [:]
|
||||
if(type == "polygon"){
|
||||
shapeMap.put("type", "Polygon")
|
||||
shapeMap.put("coordinates", d.shape.coordinates)
|
||||
udata.put("shape", shapeMap)
|
||||
udata.put("update_shape", 1)
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata))
|
||||
}else if(type == "MultiPolygon"){
|
||||
def coordinates = d.shape.coordinates
|
||||
def new_coors = []
|
||||
for(def coors : coordinates){
|
||||
new_coors << [coors]
|
||||
}
|
||||
shapeMap.put("type", "MultiPolygon")
|
||||
shapeMap.put("coordinates", new_coors)
|
||||
udata.put("shape", shapeMap)
|
||||
udata.put("update_shape", 1)
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def test2dshape(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("street","花亭路街道")
|
||||
// q.put("shape",new BasicDBObject('$exists', true))
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100)
|
||||
for(def obj : dbcursor){
|
||||
try{
|
||||
TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj)
|
||||
}catch(Exception e){
|
||||
def coordinates = obj.shape.coordinates
|
||||
for(def coors : coordinates){
|
||||
for (int i = 0; i < coors.size(); i++) { // 改为索引遍历
|
||||
def coorsArr = coors[i]
|
||||
def index = 0
|
||||
def new_coors = []
|
||||
for(def points : coorsArr){
|
||||
index ++
|
||||
if(!new_coors.contains(points)){
|
||||
new_coors << points
|
||||
}else{
|
||||
if(index<=coorsArr.size()){
|
||||
new_coors << points
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(new_coors.size()!=coorsArr.size()){
|
||||
coors[i] = new_coors
|
||||
println "[修改] "+ obj.city +" "+obj.street +" 原:"+coorsArr.size()+" 现:"+new_coors.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
try{
|
||||
TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj)
|
||||
}catch(Exception ex){
|
||||
println "仍有问题:"+obj.city +" "+obj.street+" "+ex.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def updateProvinceCenter(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\省会城市.xlsx",{d->
|
||||
def city = d['C']
|
||||
if(city){
|
||||
def q = new BasicDBObject("type","城市").append("city_short", city)
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject("capital_city", "是")
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateCityLevel(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市等级.xlsx",{d->
|
||||
def city = d['A']
|
||||
if(city){
|
||||
def q = new BasicDBObject("type","城市").append("city_short", city)
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject("city_level", d['B'])
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println city +" 没找到"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateCityNum(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市排序.xlsx",{d->
|
||||
def city = d['A']
|
||||
if(city){
|
||||
def q = new BasicDBObject("type","城市").append("city_short", city)
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject("city_position", d['B'].toString().toDouble().toInteger())
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println city +" 没找到"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-2
@@ -18,16 +18,17 @@ class GaodeCityRange {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
|
||||
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
|
||||
def citys = ["新星","胡杨河","白杨"]
|
||||
HttpRequest request = new HttpRequest()
|
||||
def rows = []
|
||||
|
||||
citys.each{c->
|
||||
if(!citys1.contains(c)){
|
||||
c = c.replace("州","")
|
||||
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
|
||||
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=66e6771767ab293ace441553c36621c1&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
|
||||
if(resp.containsKey("districts")){
|
||||
def datas = resp.get("districts")
|
||||
for(def d : datas){
|
||||
|
||||
+134
-11
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
@@ -22,6 +23,11 @@ class RegionPointCrawl {
|
||||
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static DBCollection TOWN_CENTERCOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_center")
|
||||
|
||||
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
|
||||
@@ -40,8 +46,12 @@ class RegionPointCrawl {
|
||||
// println "区域围栏 完成"
|
||||
// }
|
||||
|
||||
task.statSame()
|
||||
// task.statSame()
|
||||
|
||||
//"福建省","河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","山东省","湖北省","湖南省","江西省","江苏省","安徽省"
|
||||
def provinces = ["河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","青海省","山东省","福建省","浙江省","台湾省","河南省","湖北省","湖南省","江西省","江苏省","安徽省","广东省","海南省","四川省","贵州省","云南省","北京市","上海市","天津市","重庆市","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
task.getTownshipCenter(provinces)
|
||||
task.addTownCenter(provinces)
|
||||
}
|
||||
|
||||
def crawlCityAndDistrictCode(def province){
|
||||
@@ -210,7 +220,7 @@ class RegionPointCrawl {
|
||||
jo = JSONObject.fromObject(html)
|
||||
if(jo.infocode!="10000"){
|
||||
errorCount ++
|
||||
Thread.sleep(30*1000)
|
||||
Thread.sleep(3*1000)
|
||||
// println " 不可用Key: "+key
|
||||
}else{
|
||||
errorCount = 100
|
||||
@@ -228,15 +238,15 @@ class RegionPointCrawl {
|
||||
|
||||
static String getRandomKey2() {
|
||||
def KeyList = [
|
||||
"2af0b8533415a612b98d59bb93d664b9",
|
||||
"0246c57bbcf24aa246c207b3c99dfe41",
|
||||
"042328b3f0fa5daf426802d379521d87",
|
||||
"055d4f1b5b64a2bce18f4fe1f15c2639",
|
||||
"d5484f5170502405d88bbaffe1b496f0",
|
||||
"e74ccc4258079d2200199695110cceb1",
|
||||
"f58629fc799fbc8537ca85046c6d44a5",
|
||||
"291073f17ee0b963ccb1927ed92bf265",
|
||||
"2a1fb75990f2b39f6221e50892dd3988"
|
||||
// "2af0b8533415a612b98d59bb93d664b9",
|
||||
// "0246c57bbcf24aa246c207b3c99dfe41",
|
||||
// "042328b3f0fa5daf426802d379521d87",
|
||||
// "055d4f1b5b64a2bce18f4fe1f15c2639",
|
||||
// "d5484f5170502405d88bbaffe1b496f0",
|
||||
// "e74ccc4258079d2200199695110cceb1",
|
||||
// "f58629fc799fbc8537ca85046c6d44a5",
|
||||
"eb69e4ad2a9cdd31f163a0ed32effe6e"
|
||||
// "2af0b8533415a612b98d59bb93d664b9"
|
||||
]
|
||||
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
@@ -300,4 +310,117 @@ class RegionPointCrawl {
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def getTownshipCenter(provinces){
|
||||
for(def p : provinces){
|
||||
def query = new BasicDBObject("province", p)
|
||||
def count = TOWN_CENTERCOLL.count(query)
|
||||
if(count){
|
||||
println p +" 已存在"
|
||||
}else{
|
||||
def ret = polygon(p, "3", "base");
|
||||
if(ret){
|
||||
def citys = ret[0].districts
|
||||
if(citys){
|
||||
def allRets = []
|
||||
for(def c : citys){
|
||||
def cityTowns = getTownByCity(p, c.name,c.districts)
|
||||
allRets.addAll(cityTowns)
|
||||
}
|
||||
saveTownCenter(allRets)
|
||||
println p +" 保存:"+allRets.size()
|
||||
}
|
||||
}else{
|
||||
println p+ "无结果"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getTownByCity(prov, city,distDatas){
|
||||
def rets = []
|
||||
for(def d : distDatas){
|
||||
def townDatas = d.districts
|
||||
for(def town : townDatas){
|
||||
town.province = prov
|
||||
town.district = d.name
|
||||
town.city = city
|
||||
rets << town
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def saveTownCenter(allRets){
|
||||
for(def r : allRets){
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("province", r.province)
|
||||
obj.put("city", r.city)
|
||||
obj.put("district", r.district)
|
||||
obj.put("town", r.name)
|
||||
obj.put("center", r.center)
|
||||
obj.put("level", r.level)
|
||||
TOWN_CENTERCOLL.save(obj)
|
||||
}
|
||||
}
|
||||
|
||||
def addTownCenter(provinces){
|
||||
for(def p : provinces){
|
||||
def q = new BasicDBObject("province",p).append("type","乡镇街道")
|
||||
// q.put("location", new BasicDBObject('$exists', false))
|
||||
def dbcursor = administrative_mapping_coll.find(q)
|
||||
for(def obj : dbcursor){
|
||||
|
||||
def tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", obj.street)
|
||||
def dbo = TOWN_CENTERCOLL.findOne(tq)
|
||||
if(!dbo){
|
||||
def new_street = obj.street.replace("管理处","")
|
||||
tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", new_street)
|
||||
dbo = TOWN_CENTERCOLL.findOne(tq)
|
||||
}
|
||||
if(dbo){
|
||||
def arrs = dbo.center.split(",")
|
||||
def location = [:]
|
||||
location["lon"] = arrs[0].toDouble()
|
||||
location["lat"] = arrs[1].toDouble()
|
||||
if(!obj.location){
|
||||
def udata = new BasicDBObject("location", location)
|
||||
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
println("【修改】"+obj.city +" "+ obj.district+" "+obj.street)
|
||||
}else{
|
||||
def distance = com.fangjia.script.util.GeoUtils.calculateDistance(location.lat, location.lon, obj.location.lat, obj.location.lon);
|
||||
if(distance>1000){
|
||||
def udata = new BasicDBObject("location", location)
|
||||
println(obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 经纬度中心点相距1公里以外 原:"+obj.location.lon+","+obj.location.lat+" 新:"+dbo.center+" 距离:"+distance)
|
||||
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}else{
|
||||
|
||||
def esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street)
|
||||
if(!esobj){
|
||||
def short_prov = obj.province.replace("省", "")
|
||||
if(obj.street.contains(short_prov)){
|
||||
esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street.replace(short_prov, ""))
|
||||
}
|
||||
}
|
||||
if(esobj){
|
||||
|
||||
def location = [:]
|
||||
location["lon"] = esobj.location.lon.toDouble()
|
||||
location["lat"] = esobj.location.lat.toDouble()
|
||||
def udata = new BasicDBObject("location", location)
|
||||
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
println("【修改】"+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street+" "+ esobj.cat+" "+esobj.name +" es poi ")
|
||||
}else{
|
||||
println(obj.province+" "+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 没有找到")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
class NonghangDataExport {
|
||||
|
||||
static main(args) {
|
||||
def task = new NonghangDataExport();
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = ["ÁÙÒÊ","¹ãÖÝ", "ÉîÛÚ"]
|
||||
for(def c : citys){
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-3
@@ -2,6 +2,7 @@ package com.fangjia.script.linxsh.util
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
@@ -47,6 +48,8 @@ class AddressUtil {
|
||||
|
||||
private static final cityMap = new HashMap<String, String>()
|
||||
|
||||
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
|
||||
static {
|
||||
initCityMap()
|
||||
}
|
||||
@@ -55,9 +58,7 @@ class AddressUtil {
|
||||
* 初始化城市数据
|
||||
*/
|
||||
static def initCityMap() {
|
||||
Mongo mongo = new Mongo("mongo.db.fangjia.com", 28017);
|
||||
DBCollection SERVER_AREA = mongo.getDB("fangjia").getCollection("server_area");
|
||||
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1));
|
||||
DBCursor cursor = SERVER_AREA_COLL.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1));
|
||||
for(DBObject dbObject : cursor) {
|
||||
if(!"全国".equals(dbObject.display)) {
|
||||
cityMap.put(dbObject.display, dbObject.display)
|
||||
|
||||
@@ -239,7 +239,7 @@ public class CommonUtils {
|
||||
println(cmd)
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("category", "sell")
|
||||
data.put("priority", 3)
|
||||
data.put("priority", 1)
|
||||
data.put("status", 0)
|
||||
data.put("need_redraw_trend", needRedrawTrend)
|
||||
data.put("remark", 'week')
|
||||
|
||||
@@ -1548,6 +1548,34 @@ public class OpenApiClient {
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject houseAction(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("token", TOKEN);
|
||||
params.put("__refresh_cache","1")
|
||||
|
||||
String content = new HttpRequest().get("https://open.fangjia.com/house/around", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/house/around", params, "utf-8")
|
||||
println("${city} ${address} ³ÌÐòµ÷ÓÃÒì³£")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if (result.getInt("code") == 0) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${city} ${address}")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${address} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+15
-7
@@ -21,15 +21,15 @@ public class DistrictMonthlyCalc{
|
||||
|
||||
|
||||
public static void main(String[] args) {
|
||||
List<String> cities = ["ÉϺ£"];
|
||||
def allCity = CommonUtils.findAllCity()
|
||||
cities.addAll(allCity)
|
||||
List<String> cities = ["ÏåÑô"];
|
||||
// def allCity = CommonUtils.findAllCity()
|
||||
// cities.addAll(allCity)
|
||||
def districts = []
|
||||
cities.each {def city->
|
||||
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("name", 1).append("region", 1)).batchSize(2000).toList()
|
||||
println(city)
|
||||
}
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(20)
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(1)
|
||||
districts.each {BasicDBObject district->
|
||||
executorService.submit(new Calculator(district))
|
||||
}
|
||||
@@ -43,7 +43,7 @@ public class DistrictMonthlyCalc{
|
||||
class Calculator extends Thread{
|
||||
|
||||
static DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat_new", "mongo.db.fangjia.com")
|
||||
final BasicDBObject district;
|
||||
|
||||
public Calculator(final BasicDBObject district){
|
||||
@@ -52,14 +52,22 @@ class Calculator extends Thread{
|
||||
|
||||
@Override
|
||||
void run() {
|
||||
REGION_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo->
|
||||
def q = new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)
|
||||
q.put("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01", "yyyy-MM-dd")))
|
||||
REGION_STAT.find(q).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo->
|
||||
Date weekend = dbo.weekend
|
||||
if(DateUtil.isLastSundayOfMonth(weekend)){
|
||||
def trend = new BasicDBObject("city", district.city)
|
||||
trend.append("region", district.region)
|
||||
trend.append("district_name", district.name)
|
||||
trend.append("date", weekend.format("yyyyMM").toInteger())
|
||||
REGION_STAT_TREND.update(trend , new BasicDBObject('$set', new BasicDBObject("avg_price", dbo.avg_price)),true,false)
|
||||
def udata = new BasicDBObject()
|
||||
if(dbo.avg_price_2b && dbo.value_type!=1){
|
||||
udata.put("avg_price", dbo.avg_price_2b)
|
||||
}else{
|
||||
udata.put("avg_price", dbo.avg_price)
|
||||
}
|
||||
REGION_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
|
||||
}
|
||||
}
|
||||
REGION_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("data", 1)).each {BasicDBObject trend->
|
||||
|
||||
Reference in New Issue
Block a user