Merge remote-tracking branch 'origin/master'

This commit is contained in:
2025-07-22 17:47:40 +08:00
27 changed files with 1608 additions and 79 deletions
@@ -29,7 +29,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
@Override @Override
void process(Object args) { void process(Object args) {
List<String> cities = []; List<String> cities = [];
def allCity = CommonUtils.findAllCity() def allCity = ["广州","深圳","临沂"]//CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
cities.addAll(allCity) cities.addAll(allCity)
@@ -38,7 +38,7 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList() districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList()
println(city) println(city)
} }
ExecutorService executorService = Executors.newFixedThreadPool(20) ExecutorService executorService = Executors.newFixedThreadPool(1)
districts.each {BasicDBObject district-> districts.each {BasicDBObject district->
executorService.submit(new Calculator(district)) executorService.submit(new Calculator(district))
} }
@@ -65,22 +65,26 @@ class Calculator extends Thread{
} }
def statMap = [:] def statMap = [:]
DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo-> DISTRICT_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type).append("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01","yyyy-MM-dd")))).addOption(Bytes.QUERYOPTION_SLAVEOK).sort(new BasicDBObject("weekend", 1)).each { BasicDBObject dbo->
Date weekend = dbo.weekend Date weekend = dbo.weekend
if(DateUtil.isLastSundayOfMonth(weekend)){ if(DateUtil.isLastSundayOfMonth(weekend)){
def trend = new BasicDBObject("city", district.city) def trend = new BasicDBObject("city", district.city)
trend.append("region", district.region) trend.append("region", district.region)
trend.append("district_name", district.name) trend.append("district_name", district.name)
trend.append("estate_type", estate_type) // trend.append("estate_type", estate_type)
trend.append("date", weekend.format("yyyyMM").toInteger()) trend.append("date", weekend.format("yyyyMM").toInteger())
def udata = new BasicDBObject(); def udata = new BasicDBObject();
udata.put("avg_price", dbo.avg_price) if(dbo.avg_price_2b){
udata.put("credit", dbo.credit) // udata.put("avg_price", dbo.avg_price)
udata.put("m_date", new Date()) udata.put("avg_price", dbo.avg_price_2b)
DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false) udata.put("credit", dbo.credit)
statMap.put(weekend.format("yyyyMM").toInteger(), udata) udata.put("m_date", new Date())
DISTRICT_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
statMap.put(weekend.format("yyyyMM").toInteger(), udata)
}
} }
} }
DISTRICT_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).sort(new BasicDBObject("date", 1)).each {BasicDBObject trend-> DISTRICT_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name).append("estate_type", estate_type)).sort(new BasicDBObject("date", 1)).each {BasicDBObject trend->
@@ -88,14 +92,14 @@ class Calculator extends Thread{
def monDate = getMomDate(date) def monDate = getMomDate(date)
def yoyDate = getYoyDate(date) def yoyDate = getYoyDate(date)
BasicDBObject monDbo = statMap.get(monDate) BasicDBObject monDbo = statMap.get(monDate)
if(monDbo != null && monDbo.avg_price != 0) { if(monDbo != null && monDbo.avg_price_2b != 0) {
trend.mom = DataUtil.roundDouble(((trend.avg_price - monDbo.avg_price)/(double)monDbo.avg_price),4) trend.mom = DataUtil.roundDouble(((trend.avg_price_2b - monDbo.avg_price_2b)/(double)monDbo.avg_price_2b),4)
}else{ }else{
trend.mom = 0d trend.mom = 0d
} }
BasicDBObject yoyDbo = statMap.get(yoyDate) BasicDBObject yoyDbo = statMap.get(yoyDate)
if(yoyDbo != null && yoyDbo.avg_price != 0){ if(yoyDbo != null && yoyDbo.avg_price != 0){
trend.yoy = DataUtil.roundDouble(((trend.avg_price - yoyDbo.avg_price)/(double)yoyDbo.avg_price),4) trend.yoy = DataUtil.roundDouble(((trend.avg_price_2b - yoyDbo.avg_price_2b)/(double)yoyDbo.avg_price_2b),4)
}else{ }else{
trend.yoy = 0d trend.yoy = 0d
} }
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
* 深圳公众号成交数据录入 * 深圳公众号成交数据录入
*/ */
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1) def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年5月深圳成交明细.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -855,6 +855,32 @@ class CommonDatautils {
} }
public static def findAllDistrictsKey(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","经纬天地园二期")
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def keyMap = [:]
for(def d : dbsour){
keyMap.put(d.region+"##"+d.name, d)
}
return keyMap;
}
// public static def findAllDistrictByCity(def city, def cat, def fields){ // public static def findAllDistrictByCity(def city, def cat, def fields){
// //
// DBObject q = new BasicDBObject(); // DBObject q = new BasicDBObject();
@@ -73,6 +73,7 @@ class DistrictPriceExportBatch extends BaseBatch{
def appendResult(def headers, def ret){ def appendResult(def headers, def ret){
def sb = new StringBuffer(); def sb = new StringBuffer();
// sb.append(headers.join("@|@")) // sb.append(headers.join("@|@"))
// sb.append(System.getProperty("line.separator")); // sb.append(System.getProperty("line.separator"));
@@ -121,11 +122,16 @@ class DistrictPriceExportBatch extends BaseBatch{
def day = new Date().getDate() def day = new Date().getDate()
def startMDate = DateUtils.truncateDay(new Date()-1) def startMDate = DateUtils.truncateDay(new Date()-1)
def endMDate = DateUtils.truncateDay(new Date()) def endMDate = DateUtils.truncateDay(new Date())
if(day == 15){ if(CommonDatautils.isWindow()){
districts = getFzDistricts(city, null, null) districts = getFzDistricts(city, null, null)
}else{ } else{
districts = getFzDistricts(city, startMDate, endMDate) if(day == 15){
districts = getFzDistricts(city, null, null)
}else{
districts = getFzDistricts(city, startMDate, endMDate)
}
} }
def curPriceMap = findFzDistrictSellStat("福州",curDate) def curPriceMap = findFzDistrictSellStat("福州",curDate)
def result = [] def result = []
for(def rMap : districts){ for(def rMap : districts){
@@ -1,5 +1,6 @@
package com.fangjia.script.fuzhou package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
@@ -63,7 +64,7 @@ class DistrictSellStatBatch extends BaseBatch {
} }
def calcMonthlyPrice(){ def calcMonthlyPrice(){
//只有福州使用tob价格 //只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外
def citys = CommonUtils.findAllCity() def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1) def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
@@ -29,11 +29,11 @@ class FzDataResultStat {
static main(args) { static main(args) {
def task = new FzDataResultStat() def task = new FzDataResultStat()
task.priceMarkupRun(); // task.priceMarkupRun();
// task.regionPriceTrendRun() // task.regionPriceTrendRun()
//// task.margeExcel() //// task.margeExcel()
// task.saveCsvData() // task.saveCsvData()
// task.testNewPrice() task.testNewPrice()
} }
def margeExcel(){ def margeExcel(){
@@ -82,7 +82,7 @@ class FzDataResultStat {
} }
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) // List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"))) def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -4, "yyyyMM")))
def map2 = [:] def map2 = [:]
for(def obj : list2) { for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name def key = obj.city + "##" + obj.region + "##" + obj.name
@@ -119,7 +119,7 @@ class FzDataResultStat {
index++ index++
println row println row
} }
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") ExcelUtils.write("D:\\数据\\福州\\价格比对5_${curMonth}.xlsx", rows, "数据")
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
@@ -301,16 +301,13 @@ class FzDataResultStat {
} }
def testNewPrice(){ def testNewPrice(){
ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d-> ExcelUtils.modify("D:\\数据\\福州\\福州200个小区抽样价格核验202507.xlsx",{d->
if(d['A']){ if(d['A']){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", d['A']) q.put("comm_id", d['A'])
q.put("region",d['B'])
q.put("name",d['C'])
q.put("month", 202402)
def obj = FZ_DISTRICT_STAT_COLL.findOne(q) def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){ if(obj){
d['S'] = obj.avg_price d['D'] = obj.avg_price
} }
} }
}) })
@@ -13,11 +13,17 @@ import com.mongodb.DBCursor
import com.mongodb.DBObject import com.mongodb.DBObject
import org.apache.solr.common.util.DateUtil import org.apache.solr.common.util.DateUtil
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AddAvgpriceBatch { class AddAvgpriceBatch {
private static def citys = null; private static def citys = null;
private static def districtMap = [:]; private static def districtMap = [:];
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
static { static {
citys = ["宜昌"]//CommonDatautils.findAllCity() citys = ["宜昌"]//CommonDatautils.findAllCity()
@@ -26,10 +32,112 @@ class AddAvgpriceBatch {
static main(args) { static main(args) {
def task = new AddAvgpriceBatch(); def task = new AddAvgpriceBatch();
task.run1(); // task.run1();
// task.test() // task.test()
// task.sendPriceCommand() // task.sendPriceCommand()
// task.sendNewDistPrice() // task.sendNewDistPrice()
// task.delNoExistsDistPrice()
task.checkToBprice()
}
def checkToBprice(){
def citys = CommonDatautils.findAllCity().reverse()
for(def i=0;i<citys.size();i++){ //76
def c = citys[i]
def keyDistMap = CommonDatautils.findAllDistrictsKey(c, "district", ["city","region","name"])
def rets = findNoPrices(c, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), keyDistMap)
println c + " "+rets.size()
addCommand(rets)
}
}
def addCommand(rets){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
def m = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
CommonUtils.recalculationAverage(m.city, m.district, m.name, m.estate_type, m.weekend,false,0)
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findNoPrices(c, weekend, keyDistMap){
def q = new BasicDBObject("city", c).append("weekend", new BasicDBObject('$gte', weekend)).append("avg_price_2b", null)
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("district_name", 1)
keys.put("estate_type", 1)
keys.put("avg_price_2b", 1)
keys.put("weekend", 1)
def dbcursor = DIST_STAT_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbcursor){
def distObj = keyDistMap.get(d.region+"##"+d.district_name)
if(distObj){
if(!d.avg_price_2b) {
rets << ["city": d.city, "district": d.region, "name": d.district_name,"estate_type":d.estate_type, "weekend": d.weekend]
}
}
}
return rets
}
def delNoExistsDistPrice(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
index ++
if(index<250){
continue
}
println index +" "+c
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name"])
def nameKeys = []
for(def dist : dists){
nameKeys << dist.region+"#"+dist.name
}
delDistPrice(c, nameKeys)
}
}
def delDistPrice(city, nameKeys){
def q = new BasicDBObject("city", city)
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("district_name", 1)
def dbcursor = DIST_STAT_COLL.find(q, keys).batchSize(5000)
for(def d : dbcursor){
def key = d.region +"#"+d.district_name
if(!nameKeys.contains(key)){
DIST_STAT_COLL.remove(new BasicDBObject("_id", d._id))
println "删除:"+city +" "+ d.district_name
}
}
} }
def sendNewDistPrice(){ def sendNewDistPrice(){
@@ -1,21 +1,225 @@
package com.fangjia.script.gj package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import org.apache.poi.xwpf.usermodel.XWPFDocument
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.Semaphore import java.util.concurrent.Semaphore
import java.util.concurrent.atomic.AtomicLong
class DistrictStat { class DistrictStat {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) { static main(args) {
def task = new DistrictStat(); def task = new DistrictStat();
// task.run() // task.run()
task.run("无锡") // task.run("无锡")
// 合肥市
// 芜湖市
// 安庆市
// 马鞍山市
// 宣称市
// 六安市
// def citys = ["宣城","合肥","芜湖","安庆","马鞍山","六安"]
// for(def city : citys){
// task.stat(city)
// }
// task.cityStat()
task.cityStat1()
}
def cityStat1(){
def index = 0
def rets = []
ExcelUtils.modify("D:\\数据\\小区\\城市数量统计1.xlsx",{d->
def districts = null
if(d['D']=="义乌"){
districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region =="义乌"){
newDists << dist
}
}
districts = newDists
}else if(d['D']=="金华"){
districts = CommonDatautils.findAllDistrictByCity("金华", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region !="义乌"){
newDists << dist
}
}
districts = newDists
}else if(d['D']=="无锡"){
districts = CommonDatautils.findAllDistrictByCity("无锡", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region !="江阴"){
newDists << dist
}
}
districts = newDists
}else if(d['D']=="苏州"){
districts = CommonDatautils.findAllDistrictByCity("苏州", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region !="常熟"){
newDists << dist
}
}
districts = newDists
}else if(d['D']=="济南"){
districts = CommonDatautils.findAllDistrictByCity("济南", "district", ["region","name"])
def newDists = []
for(def dist : districts){
if(dist.region !="莱芜"){
newDists << dist
}
}
districts = newDists
}else{
districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["region","name"])
}
d['F'] = districts.size()
if(districts){
d['E'] = "是"
}else{
d['E'] = "否"
}
def prov = null
if(districts){
index ++
def data = fullCityMap.get(d['D'])
if(data){
prov = data.province
}
def str = districts.collect { it.name }.join(", ")
rets << ["id":index, "type":"A", "province":prov, "city":d['D'], "names":str]
createDoc(rets)
}
}, 0)
// FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表1.dat", [], "utf-8", false);
}
def createDoc(dataList){
// 创建 Word 文档
def doc = new XWPFDocument()
def table = doc.createTable(dataList.size() + 1, 5) // 行数=数据行+表头,列数=3
// 设置表头
def headerRow = table.getRow(0)
headerRow.getCell(0).setText("序号")
headerRow.getCell(1).setText("类型")
headerRow.getCell(2).setText("省份")
headerRow.getCell(3).setText("城市")
headerRow.getCell(4).setText("楼盘名称")
// 填充数据
dataList.eachWithIndex { item, index ->
def row = table.getRow(index+1) // 跳过表头
row.getCell(0).setText(item.id.toString())
row.getCell(1).setText(item.type.toString())
row.getCell(2).setText(item.province.toString())
row.getCell(3).setText(item.city.toString())
row.getCell(4).setText(item.names.toString())
}
// 保存到文件
def outputFile = new File("A类城市楼盘列表.docx")
doc.write(new FileOutputStream(outputFile))
println "Word 表格生成成功: ${outputFile.absolutePath}"
}
def cityStat(){
def index = 0
def rets = []
ExcelUtils.modify("D:\\数据\\小区\\城市数量统计.xlsx",{d->
def districts = CommonDatautils.findAllDistrictByCity(d['D'], "district", ["city", "region", "name"])
if(districts){
d['F'] = districts.size()
d['E'] = "是"
rets << ["序号","类型","省份","城市","楼盘名称","金华"]
def data = fullCityMap.get(d['D'])
def prov = null
if(data){
prov = data.province
}
for(def dist : districts){
index++
rets << [index, "A", prov,dist.city, dist.name, dist.region]
}
}
}, 0)
// ExcelUtils.write("D:\\数据\\小区\\A类城市楼盘列表.xlsx", rets, "楼盘数据详情列表")
FileUtils.batchCreateFile(rets, "D:\\数据\\小区\\A类城市楼盘列表.dat", [], "utf-8", false);
}
def stat(city){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name","alias"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def result = Collections.synchronizedList(new ArrayList());
def countResult = Collections.synchronizedList(new ArrayList());
def rets = []
// def houseTotal = CommonDatautils.findHouseCountByDate(city, null, null, null, "sell", 20240611)
rets << city
rets << districts.size()
// rets << houseTotal
for(def i=0;i<districts.size();i++){
final def o = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = CommonDatautils.findHouseCountByDate(o.city, o.region, o.name, o.alias, "sell", 20240611)
if(count){
result << o
countResult << count
}
println totalCount.get() +" "+o.region +" "+o.name+" "+count
} catch (Exception e) {
println totalCount.get() +" 查房源Error: "+city+" "+ o.region+" "+o.name+" "+e.getMessage()
} finally {
latch.countDown()
}
};
})
}
latch.await()
rets << countResult.sum()
rets << result.size()
println rets
} }
def run(def city){ def run(def city){
def weekend = DateUtils.parseFromTightYYYYMMDD("2020-10-25") def weekend = DateUtils.parseFromTightYYYYMMDD("2020-10-25")
@@ -64,7 +64,9 @@ class FangjiatongUserStat {
def stat(){ def stat(){
def users = findUses(12) def users = findUses(12)
def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"] // def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"]
// def months = ["2020-01-01","2020-02-01","2020-03-01","2020-04-01","2020-05-01","2020-06-01","2020-07-01","2020-08-01","2020-09-01","2020-10-01","2020-11-01","2020-12-01","2021-01-01"]
def months = ["2022-01-01","2022-02-01","2022-03-01","2022-04-01","2022-05-01","2022-06-01","2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01"]
def rets = [] def rets = []
def header = ["用户id","支行名称", "备注名称"] def header = ["用户id","支行名称", "备注名称"]
months.each{m-> months.each{m->
@@ -87,7 +89,7 @@ class FangjiatongUserStat {
} }
rets << ret rets << ret
} }
ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2019") ExcelUtils.write("D:\\数据\\农行统计.xlsx", rets, "农行统计-2022")
} }
def findUses(role){ def findUses(role){
@@ -27,10 +27,29 @@ class SuzouPocTest {
static main(args) { static main(args) {
def task = new SuzouPocTest() def task = new SuzouPocTest()
// task.run() // task.run()
task.newRun() // task.newRun()
task.test()
}
def test(){
ExcelUtils.modify("D:\\数据\\上海联合产权交易所\\第1批拟处置房产清单(长城大厦、台山碧桂园).xlsx", { d ->
if(d['O']){
def floorAdds = d['I'].split("/")
def ret1 = OpenApiClient.addressEvaluation(d['O'], null, d['D'], d['D'], d['K'], floorAdds[0], floorAdds[1])
if (ret1) {
d['N'] = ret1.totalPrice
d['M'] = ret1.avgPrice
d['P'] = ret1.district
d['Q'] = ret1.name
}
}
});
} }
def newRun(){ def newRun(){
println("start:"+new Date().toString()) println("start:"+new Date().toString())
def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls" def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls"
newMatch(filePath) newMatch(filePath)
@@ -21,7 +21,7 @@ class FileUtils {
def sb = new StringBuffer(); def sb = new StringBuffer();
if(headers){ if(headers){
sb.append(headers.join("|@|")) sb.append(headers.join("@"))
sb.append("\n"); sb.append("\n");
} }
@@ -32,7 +32,7 @@ class FileUtils {
for(def row : ret){ for(def row : ret){
if(row){ if(row){
index++ index++
sb.append(StringUtils.join(row.toArray(), "|@|")) sb.append(StringUtils.join(row.toArray(), "@"))
sb.append("\n"); sb.append("\n");
if(index%10000==0){ if(index%10000==0){
@@ -0,0 +1,583 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.newDistrict.InvokeHttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
import net.sf.json.groovy.JsonSlurper
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
class SzGovHouseCrawl {
static main(args) {
def task = new SzGovHouseCrawl();
task.run();
}
def run(){
def index_content = """
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>一手房源公示</title>
<link href="css/marster.css" rel="stylesheet" type="text/css" />
</head>
<body>
<form name="Form1" method="post" action="index.aspx" id="Form1">
<div>
<input type="hidden" name="__EVENTTARGET" id="__EVENTTARGET" value="" />
<input type="hidden" name="__EVENTARGUMENT" id="__EVENTARGUMENT" value="" />
<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="nFQUad+408Aq5zCXFL9mwJbwfbUjHqOTwTmTIzIV/nn2wuuQIXExg/fWXnK8Ll6o8tHsGFJtGbKc87jIO8eGWmN56r/iP2F/Oi3sPuUM14rN/OFTR/y1yS4eQXiTUniZCpvc+hwWUXM6La+cm63hbAskQFo2Geb/8JeX/kCREa/vJqSoFTxJFuxoDevNRV5QeytYQ8PxV89z/L/KAGHwo7sR66wzuWciXwvLcXa6BWaA9aU953Rr1QnFrqy3QrAnmLmxfEtEbvw6cySgECeARwElfHdG0dKGni1inFrFf3lYDdNAL2gLoQoNFBIbDj/vXaYZQoAypynsqc8hiZotSa5aqehFlDxDu3TO3jRcWKQDue55UoGZURl8WyjlfLpebiAdSQirRRrLRQHcb4Baf7U0y/JiFFk0A75I9bfJM72eq16jjjhV9++DNXJRUn5KGMSn5h+PxwcsS8rSga0KRwsM9LDylCsNkedVucDhMCGrNwmbhAsnUIiMD7Va8wQizQllRtDJWPFZRU+YPUWxqRkfJnTEzdl8MQw7eEC6rweywfrDn3VLvw8RkRqAdaSyA5K3z4VXR0QVuBcX6teex3UIQQep9oWVvK6FGlpHEfkqEPYdwnyoEoGgRJHk/R0xhCx2ahrlYI04dbT2GaugM2JDVBnGWhQBYeylLukCVpBnaxUgiGG1Dx6lUBbLi0yFt3GkdYzi5XeTzQoGsMTbdVMBvcIqqFOs9dZBfebrWrpwTnAgUldkrtyw6j9Lf0sKPCUifvCCg8Ks1OnniSkpI1zJ0sfseGrnPQv7357IEg6UWyfvMFr3jjzqcX0QV35ajJMpzfaKIYGlzzIkQwEfjhNU83+nhI3yct8nyfmE2lROKDzOdCYFmlOPVQ1yF0vdPWs2cmx3SeQl53r1nPokHRQFuYKif4B7Tx/GN/1jPoKtrqBekL/8+ewdF1YAL8z7okg4LQ7DWaITUPvZmP7p/kAqilgKVMzqAOmdl5FzK1RwDn1fh6Ciw79aM3MPYwzkfxwA9S9uo67RuF4DHbVkuob/SJ1JGfXvFNJEvm/45UJWBFKEYWnmJJLvbeZSSZ455ox0aNLtzsTpTm3yr7S+LVNqTQIxEdFMed1gC/9caV753lkuEAHlxWwZHE3eULOw04aM8w==" />
</div>
<script type="text/javascript">
//<![CDATA[
var theForm = document.forms['Form1'];
if (!theForm) {
theForm = document.Form1;
}
function __doPostBack(eventTarget, eventArgument) {
if (!theForm.onsubmit || (theForm.onsubmit() != false)) {
theForm.__EVENTTARGET.value = eventTarget;
theForm.__EVENTARGUMENT.value = eventArgument;
theForm.submit();
}
}
//]]>
</script>
<script src="/ris/WebResource.axd?d=oAb-PGCUASjSAEHmsWLcW3nRSbp0kTk2wHZPA_xOoR1TX097F_c8lWeB0anDU_zTJ7nGX_3dSaDy8zzgj1ZXo1L1Sc41&amp;t=638314433260000000" type="text/javascript"></script>
<script src="/ris/ScriptResource.axd?d=ez7z8Rk-sRUjmvQcOq01fhxlJ1QkOJwcsZyID_Ni3ynenZjRnecb_-APpFxKSgOMhcW8F7rusL02bjz8mZvC6mdZ5fFUnv7hA8P09LlmyTRz8AzNNpmFk65QF_tFzWpkOb3lJg2&amp;t=636978664134464033" type="text/javascript"></script>
<script src="/ris/ScriptResource.axd?d=3ACyCjbvPx1mr6XvFjwUpUrueEjvCnwM6jay2gUKFWC2bAv9PJDw7_w8e-FijIouXVInB0UX5vw0I3Fbx5YA92gSlMgpLLv3eu8tE_KjYAT1gVhsDve7JfQShCjQ8wBnrY5v-Q2&amp;t=636978664134464033" type="text/javascript"></script>
<div>
\t<input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="2A35A6B2" />
\t<input type="hidden" name="__VIEWSTATEENCRYPTED" id="__VIEWSTATEENCRYPTED" value="" />
\t<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="c/yjhDZvdy6HmhcL4EkG1J1Z3q6SMKFIycFC2eMEcd9PHzbGDxG56wrvw1REP793VkDqjnfrk1++ArpjUL0ql/yAB3MB5SUwn7h0YzAXw3/YG2ipVYdaOarImjt0o8JYuszLTaNRVxXmK8KTgUXOhpL0mroOqigxEaXUaeSKml9LBl2xzJttjM+uErnLyCLXSyLW0qjXWs+pCPqD5SgIhBUuVmg=" />
</div>
<div class="wrap">
<div class="record fix mb10">
<script type="text/javascript">
//<![CDATA[
Sys.WebForms.PageRequestManager._initialize('scriptManager2', document.getElementById('Form1'));
Sys.WebForms.PageRequestManager.getInstance()._updateControls(['tupdatepanel2'], [], [], 90);
//]]>
</script>
<div class="recordLinkbox recordLink">
所在区:<a id="hypAll" href="javascript:__doPostBack('hypAll','')" style="color:Red;">全市</a>
<a id="hypBa" href="javascript:__doPostBack('hypBa','')" style="color:Black;">宝安</a>
<a id="hypFt" href="javascript:__doPostBack('hypFt','')" style="color:Black;">福田</a>
<a id="hypLg" href="javascript:__doPostBack('hypLg','')" style="color:Black;">龙岗</a>
<a id="hypLh" href="javascript:__doPostBack('hypLh','')" style="color:Black;">罗湖</a>
<a id="hypNs" href="javascript:__doPostBack('hypNs','')" style="color:Black;">南山</a>
<a id="hypQh" href="javascript:__doPostBack('hypQh','')" style="color:Black;">前海</a>
<a id="hypYt" href="javascript:__doPostBack('hypYt','')" style="color:Black;">盐田</a>
<a id="hypLhQ" href="javascript:__doPostBack('hypLhQ','')" style="color:Black;">龙华</a>
<a id="hypGm" href="javascript:__doPostBack('hypGm','')" style="color:Black;">光明</a>
<a id="hypPs" href="javascript:__doPostBack('hypPs','')" style="color:Black;">坪山</a>
<a id="hypDp" href="javascript:__doPostBack('hypDp','')" style="color:Black;">大鹏</a>
<a id="hypSSHZ" href="javascript:__doPostBack('hypSSHZ','')">深汕合作</a></div>
<div class="deveBox" style="line-height: 0px;">
<p class="left mr25">
项目查询 <input name="tep_name" type="text" id="tep_name" style="height:28px;width:150px;" /></p>
<p class="left mr25">
开发企业 <input name="organ_name" type="text" value="深圳市城建产业园发展有限公司" id="organ_name" style="height:28px;width:150px;" /></p>
<p class="left mr25">
宗地地址 <input name="site_address" type="text" id="site_address" style="height:28px;width:150px;" /></p>
</div>
<input type="submit" name="Button1" value="查询" id="Button1" style="background: #1956c6;
width: 75px; height: 28px; line-height: 28px; display: block; text-align: center; cursor:pointer;
font-size: 14px; color: #fff;border-radius: 2px;" />
</div>
<div id="updatepanel2">
\t
<div class="record fix">
<div class="fix">
<table class="table ta-c bor-b-1 table-white">
<tr>
<th style="width: 92px;">
序号
</th>
<th style="width: 220px;">
预售证号
</th>
<th style="width: 312px;">
项目名称
</th>
<th style="width: 270px;">
开发企业
</th>
<th style="width: 110px;">
所在区
</th>
<th style="width: 132px;">
批准时间
</th>
</tr>
<tr class="">
<td>
1
</td>
<td>
<a href='./certdetail.aspx?id=133423' target='_self'>
深房许字(2024)罗湖002号</a>
</td>
<td>
<a href='projectdetail.aspx?id=133423' target='_self'>
凌云华府</a>
</td>
<td>
深圳市城建产业园发展有限公司
</td>
<td>
罗湖
</td>
<td>
2024-06-27
</td>
</tr>
<tr class="bg-grayf7">
<td>
2
</td>
<td>
<a href='./certdetail.aspx?id=124226' target='_self'>
深房许字(2022)罗湖007号</a>
</td>
<td>
<a href='projectdetail.aspx?id=124226' target='_self'>
高投大厦</a>
</td>
<td>
深圳市城建产业园发展有限公司
</td>
<td>
罗湖
</td>
<td>
2022-12-16
</td>
</tr>
<tr style="height:18px;">
<td colspan="6">
</td>
</tr>
</tr></table>
</div>
<div class="page">
<div class="pages pagehtml right">
\t<!-- AspNetPager V7.0.2 for VS2005 & VS2008 Copyright:2003-2007 Webdiyer (www.webdiyer.com) -->
\t<!--记录总数只有一页,AspNetPager已自动隐藏,若需在只有一页数据时显示AspNetPager,请将AlwaysShow属性值设为true!-->
\t<!-- AspNetPager V7.0.2 for VS2005 & VS2008 End -->
\t
</div>
</div>
</div>
</div>
</div>
</div>
<script type="text/javascript">
//<![CDATA[
Sys.Application.initialize();
//]]>
</script>
</form>
</body>
</html>
"""
getRoomRefDetail("https://zjj.sz.gov.cn/ris/bol/szfdc/housedetail.aspx?id=1979350",[:])
def doc = Jsoup.parse(index_content);
Elements trs = doc.select("table tr")
def name = "深圳市城建产业园发展有限公司"
def results = []
def header = ["楼层","房间号","单元号","销售状态","楼栋名称","预售许可证号","项目名称","所属项目地址","开发商名称","开盘日期","监管银行","监管账户名称","监管账户账号","房屋用途","户型","预测总建筑面积","预测套内建筑面积","预测公摊建筑面积","实际总建筑面积","实际套内建筑面积","实际公摊建筑面积","城市","拟售价格-按建筑面积计","拟售价格-按套内建筑面积计","预售总套数","现售总套数","栋数"]
results << header
//开发商查询结果页
println trs.size()-2 +" 预售证"
for(def i=1 ;i<trs.size();i++){
def href1 = trs[i].select("td:eq(1) a").attr("href")
if(href1){
href1 = "https://zjj.sz.gov.cn/ris/bol/szfdc"+ href1.substring(1)
def href2 = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+trs[i].select("td:eq(2) a").attr("href")
def sellObjMap = getSellCodeObject(href1) //预售证页面
println i+ " 预售证页面 加载完成"
def projectMap = getProjectDetail(href2) //抓取项目页面
println i+ " 预售证页面 抓取项目页面"
if(projectMap){
def projList = projectMap.projList
for(def pro : projList){
def nowsellrooms = pro.yusell_roomDetails
for(def yusell : nowsellrooms){
def row = []
row << yusell["楼层"]
row << yusell["房号"]
row << yusell["座号"]
row << yusell["房号状态"]
row << pro["楼名"]
row << sellObjMap['许可证号']
row << sellObjMap['项目名称']
row << sellObjMap['所在位置']
row << sellObjMap['发展商']
row << sellObjMap['发证日期 (开始销售日期)']
row << projectMap['价款监管机构1']
row << projectMap['账户名称']
row << projectMap['账号']
row << yusell["用途"]
row << ""
row << yusell["预售-建筑面积"]
row << yusell["预售-套内建筑面积"]
row << yusell["预售-分摊面积"]
row << yusell["竣工-建筑面积"]
row << yusell["竣工-套内建筑面积"]
row << yusell["竣工-分摊面积"]
row << "深圳"
row << yusell["拟售价格-按建筑面积计"]
row << yusell["拟售价格-按套内建筑面积计"]
row << projectMap['预售总套数']
row << projectMap['现售总套数']
row << sellObjMap["栋数"]
results << row
}
}
}
}
}
ExcelUtils.write("D:\\数据\\新房\\${name}.xlsx", results, "深圳")
}
def getSellCodeObject(url){
String content = new HttpRequest().get(url, null, "utf-8")
def doc = Jsoup.parse(content);
Elements trs = doc.select("table tr")
def sellObjMap = [:]
def sellUseList = []
for(def i=1 ;i<trs.size();i++){
def tds = trs[i].select("td")
if(tds.size()==4){
sellObjMap.put(tds[0].text(), tds[1].text())
sellObjMap.put(tds[2].text(), tds[3].text())
}else if(tds.size()==6){
if(tds[1].text()!="--"){
def map = [:]
map.put(tds[0].text(), tds[1].text())
map.put(tds[2].text(), tds[3].text())
map.put(tds[4].text(), tds[5].text())
sellUseList << map
}
}else if(tds.size()==2){
sellObjMap.put(tds[0].text(), tds[1].text())
}
}
sellObjMap.useList = sellUseList
return sellObjMap;
}
def getProjectDetail(url){
String content = new HttpRequest().get(url, null, "utf-8")
def doc = Jsoup.parse(content);
// def sellInfoUrl = doc.select("strong:contains(卖方信息) a").attr("href")
// def devMap = getSellInfo("https://zjj.sz.gov.cn/ris/bol/szfdc/"+sellInfoUrl)
Elements tableEle = doc.select("table")
Elements trs1 = tableEle.get(0).select("tr")
def projMap = [:]
for(def i=1 ;i<trs1.size();i++) {
def tds = trs1[i].select("td")
for (def j = 0; j+2 <= tds.size(); j=j+2) {
projMap.put(tds[j].text(), tds[j + 1].text())
}
}
Elements trs2 = tableEle.get(1).select("tr")
def projList = []
for(def i=1 ;i<trs2.size();i++) {
def tds = trs2[i].select("td")
def map = [:]
map.put("项目名称", tds[0].text())
map.put("楼名", tds[1].text())
map.put("建设工程规划许可证", tds[2].text())
map.put("建筑工程施工许可证", tds[3].text())
def roomUrl = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+tds[4].select("a").attr("href").replace("&amp;","&")
map.put("套房信息", roomUrl)
def yusellroomList = getYuSellRoomDetail(roomUrl)
map.yusell_roomDetails = yusellroomList
// def nowsellroomList = getNowSellRoomDetail(roomUrl)
// map.nowsellroomDetails = nowsellroomList
projList << map
}
projMap.projList = projList
// projMap.devInfoMap = devMap
return projMap
}
def getYuSellRoomDetail(roomUrl){
String content = new HttpRequest().get(roomUrl, null, "utf-8")
def doc = Jsoup.parse(content);
def roomList = []
// Elements tdsEle = doc.select("td:contains(房号:)")
// for(def td : tdsEle){
// def tdText = td.text()
// if(tdText.contains("房号")){
// def roomMap = [:]
// def divEle = td.select("div")
// roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
// roomMap.put("房号状态", divEle.get(1).text())
// def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+divEle.get(1).select("a").attr("href")
// roomMap.put("房号url", href)
// roomMap = getRoomRefDetail(href, roomMap)
// roomList << roomMap
// }
// }
def hrefs = doc.select("#divShowBranch a")
for(def hrefObj : hrefs){
def danyuan = hrefObj.text().replace("[", "").replace("]", "")
def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/" + hrefObj.attr("href")
def danyuan_list = getDanyuanPage(href)
roomList.addAll(danyuan_list)
}
return roomList
}
def getDanyuanPage(roomUrl){
String content = new HttpRequest().get(roomUrl, null, "utf-8")
def doc = Jsoup.parse(content);
Elements tdsEle = doc.select("td:contains(房号:)")
def roomList = []
for(def td : tdsEle){
def tdText = td.text()
if(tdText.contains("房号")){
def roomMap = [:]
def divEle = td.select("div")
roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
roomMap.put("房号状态", divEle.get(1).text())
def href = divEle.get(1).select("a").attr("href")
if(href){
href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+ href
roomMap.put("房号url", href)
roomMap = getRoomRefDetail(href, roomMap)
}
roomList << roomMap
}
}
return roomList
}
def getContent(href){
String content = null
def error = 0
while (error<=5){
try{
content = new HttpRequest().get(href, null, "utf-8")
if(content){
error = 100
}else{
Thread.sleep(3000)
error ++
}
}catch(Exception e){
println e.getMessage() + " "+ href
Thread.sleep(3000)
error ++
}
}
return content
}
def getRoomRefDetail(href, infoMap){
String content = getContent(href)
def doc = Jsoup.parse(content);
Elements trsEle = doc.select("table tr")
println href +" "+trsEle.size()
if(trsEle.size()==0){
def error = 0
while (error<=3){
Thread.sleep(3000)
content = new HttpRequest().get(href, null, "utf-8")
doc = Jsoup.parse(content);
trsEle = doc.select("table tr")
println href +" "+trsEle.size()
if(trsEle.size()>0){
error = 100
}else{
error ++
}
}
}
for(def i=0;i<=1;i++){
def tds = trsEle[i].select("td")
for (def j = 0; j+2 <= tds.size(); j=j+2) {
infoMap.put(tds[j].text(), tds[j + 1].text())
}
}
def tds1 = trsEle.get(2).select("td")
infoMap.put("拟售价格-按建筑面积计", tds1[1].text().replace("--元/平方米","").replace("(按建筑面积计)",""))
infoMap.put("拟售价格-按套内建筑面积计", tds1[2].text().replace("--元/平方米","").replace("(按套内建筑面积计)",""))
infoMap.put("合同号", tds1[4].text())
def tds2 = trsEle.get(4).select("td")
for (def j = 0; j+2 <= tds2.size(); j=j+2) {
infoMap.put("预售-"+tds2[j].text(), tds2[j + 1].text().replace("平方米",""))
}
def tds3 = trsEle.get(6).select("td")
for (def j = 0; j+2 <= tds3.size(); j=j+2) {
infoMap.put("竣工-"+tds3[j].text(), tds3[j + 1].text().replace("平方米",""))
}
return infoMap;
}
def getSellInfo(url){
String content = new HttpRequest().get(url, null, "utf-8")
def doc = Jsoup.parse(content);
Elements trs1 = doc.select("table tr")
def devMap = [:]
for(def i=1 ;i<trs1.size();i++) {
def tds = trs1[i].select("td")
for (def j = 0; j+2 <= tds.size(); j=j+2) {
devMap.put(tds[j].text(), tds[j + 1].text())
}
}
return devMap
}
def getNowSellRoomDetail(url){
// def doc = Jsoup.parse(content);
// def header = [:]
// def VIEWSTATE = doc.select("#__VIEWSTATE").attr("value")
// def VIEWSTATEGENERATOR = doc.select("#__VIEWSTATEGENERATOR").attr("value")
// def EVENTVALIDATION = doc.select("#__EVENTVALIDATION").attr("value")
// doc.select("#__VIEWSTATEGENERATOR").attr("value")
// header.put("__VIEWSTATE", doc.select("#__VIEWSTATE").attr("value"))
// header.put("__EVENTTARGET", "imgBt2")
// header.put("__VIEWSTATEGENERATOR", doc.select("#__VIEWSTATEGENERATOR").attr("value"))
// header.put("__EVENTVALIDATION", doc.select("#__EVENTVALIDATION").attr("value"))
//// params.put("scriptManager1","updatepanel1|imgBt2")
//// String res = InvokeHttpRequest.post(url, params, "utf-8");
// getRoomNowSellTabInfo(url, VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION)
//
// def params = [:]
// params.put("id", "49636")
// params.put("presellid", "129765")
// HttpRequest re = new HttpRequest();
// String res = re.post(url, params, header, "utf-8");
String res = new HttpRequest().get(url+"&Branch=未命名&isBlock=xs", null, "utf-8")
def doc1 = Jsoup.parse(res);
Elements tdsEle = doc1.select("td:contains(房号:)")
def roomList = []
for(def td : tdsEle){
def tdText = td.text()
if(tdText.contains("房号")){
def roomMap = [:]
def divEle = td.select("div")
roomMap.put("房号", divEle.get(0).text().replace("房号:", ""))
roomMap.put("房号状态", divEle.get(1).text())
def href = "https://zjj.sz.gov.cn/ris/bol/szfdc/"+divEle.get(1).select("a").attr("href")
roomMap.put("房号url", href)
roomMap = getRoomRefDetail(href, roomMap)
roomList << roomMap
}
}
return roomList
}
def getRoomNowSellTabInfo(targetUrl, VIEWSTATE, VIEWSTATEGENERATOR, EVENTVALIDATION){
try {
// 创建连接
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
// 基础配置
conn.setRequestMethod("POST");
conn.setDoOutput(true); // 允许写入输出流
conn.setConnectTimeout(5000); // 5秒连接超时
conn.setReadTimeout(10000); // 10秒读取超时
// 设置请求头(模拟浏览器)
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9");
// 构建Form Data参数(带中文需编码)
String formData = "__VIEWSTATE=" + URLEncoder.encode(VIEWSTATE, "UTF-8") +
"&__VIEWSTATEGENERATOR=" + URLEncoder.encode(VIEWSTATEGENERATOR, "UTF-8") +
"&__EVENTVALIDATION="+URLEncoder.encode(EVENTVALIDATION, "UTF-8") +
"&__EVENTTARGET=imgBt2";
// 发送请求体
OutputStream os = conn.getOutputStream()
byte[] input = formData.getBytes("utf-8");
os.write(input, 0, input.length);
// 获取响应状态码
int status = conn.getResponseCode();
System.out.println("响应状态码: " + status);
String responseLine;
// 读取响应内容
if (status == HttpURLConnection.HTTP_OK) {
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"))
StringBuilder response = new StringBuilder();
while ((responseLine = br.readLine()) != null) {
response.append(responseLine.trim());
}
System.out.println("响应内容:\n" + response.toString());
} else {
System.out.println("请求失败,错误码: " + status);
}
return responseLine
} catch (Exception e) {
e.printStackTrace();
}
return null
}
}
@@ -3,10 +3,12 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.apache.commons.lang3.SerializationUtils
import java.util.regex.Pattern import java.util.regex.Pattern
@@ -14,15 +16,30 @@ class AddrAlaisProcess {
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
def fullCityMap = CommonDatautils.queryFullNameCity() def fullCityMap = CommonDatautils.queryFullNameCity()
//
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) { static main(args) {
def task = new AddrAlaisProcess() def task = new AddrAlaisProcess()
// task.run() // task.run()
// task.splitDist() // task.splitDist()
// task.addAddr() // task.addAddr()
task.nineApi() // task.nineApi()
// task.createResult() // task.createResult()
// task.mergeResult() // task.mergeResult()
// task.addLocation()
task.updateAddr()
}
def addLocation(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\地图无路.xlsx",{d->
def obj = CommonDatautils.findDistrictById(d['A'])
if(obj){
d['G'] = obj.lng/10000000000.0
d['H'] = obj.lat/10000000000.0
}
})
} }
def splitDist(){ def splitDist(){
@@ -102,7 +119,7 @@ class AddrAlaisProcess {
def nineApi(){ def nineApi(){
def i = 0; def i = 0;
while(i<20) { while(i<1) {
def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx" def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx"
if (CommonDatautils.isWindow()) { if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx" fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx"
@@ -122,6 +139,16 @@ class AddrAlaisProcess {
def roadObj = findBaiduDatas( data.city, d['H']) def roadObj = findBaiduDatas( data.city, d['H'])
if(roadObj){ if(roadObj){
d['J'] = "百度存在" d['J'] = "百度存在"
}else{
// def distData = mcaDistMap.get(d['D'] + "#" + d['E'])
// def fullDistName = null
// if (distData) {
// fullDistName = distData.district
// }
// def obj = CommonDatautils.findGaoDeDatas(data.city, fullDistName, d['H'])
// if(obj){
// d['J'] = "高德存在"
// }
} }
} }
} }
@@ -208,4 +235,39 @@ class AddrAlaisProcess {
def obj = BAIDU_POI_COLL.findOne(q) def obj = BAIDU_POI_COLL.findOne(q)
return obj; return obj;
} }
def updateAddr(){
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\地址及地址别名-详情-1.xlsx",{d->
def obj = CommonDatautils.findDistrictById(d['C'])
if(obj && d['G']){
def flag = DataHandleUtils.isStandardRoad(d['L'])
def udata = new BasicDBObject()
if(obj.address == d['H']){
udata.put("address", d['L'])
}else{
if(obj.addr_alias){
def new_addr_alias = SerializationUtils.clone(obj.addr_alias)
for(def alias : new_addr_alias){
if(d['H'] == alias){
if(flag){
if(d['L']!=alias){
alias = d['L']
}
}else{
alias = ""
}
}
}
if(new_addr_alias!=obj.addr_alias){
udata.put("addr_alias", new_addr_alias)
}
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', obj._id),new BasicDBObject('$set',udata))
}
}
})
}
} }
@@ -225,15 +225,12 @@ class AutoAddOfficeBatch extends BaseBatch {
} }
def repeatDist(){ def repeatDist(){
def citys = ["南京"]//CommonDatautils.findAllCity().reverse() def citys = ["成都"]//CommonDatautils.findAllCity().reverse()
def allrets = [] def allrets = []
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"] allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
def index = 0 def index = 0
for (def c : citys) { for (def c : citys) {
index++ index++
if(["宁波","无锡","青岛"].contains(c)){
continue
}
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"]) def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
def rets = checkRepeatDistrictBatch(c, districts) def rets = checkRepeatDistrictBatch(c, districts)
println c + " " + districts.size() println c + " " + districts.size()
@@ -254,13 +251,14 @@ class AutoAddOfficeBatch extends BaseBatch {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityId = cityCodeDataMap.get(city) def cityId = cityCodeDataMap.get(city)
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502) def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202505)
def addCount = new AtomicLong() def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size()) final CountDownLatch latch = new CountDownLatch(districts.size())
def total = districts.size() def total = districts.size()
def rets = new CopyOnWriteArrayList() def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet() def sets = new CopyOnWriteArraySet()
for (def i = 0; i < total; i++) { for (def i = 0; i < total; i++) {
println i
final def o1 = districts[i] final def o1 = districts[i]
// fixedThreadPool.execute(new Runnable() { // fixedThreadPool.execute(new Runnable() {
// void run() { // void run() {
@@ -0,0 +1,86 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CommunityShapeProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
static main(args) {
def task = new CommunityShapeProcess();
task.run();
}
def run(){
def q = new BasicDBObject()
q.put("province_name", "河北省")
q.put("shape", new BasicDBObject('$exists', true))
def keys = new BasicDBObject()
keys.put("shape", 1)
def dbcursor = community_CONN.find(q, keys).batchSize(1000)
for(def d : dbcursor){
if(d.shape.type=="Polygon"){
def coordinates = d.shape.coordinates.get(0)
def points = []
def index = 0
for(def arrs : coordinates){
if(index!=0 && index+1!= coordinates.size()){
if(!points.contains(arrs)){
points << arrs
}else{
println d._id +" 重复数据:"+arrs
}
}else{
points << arrs
}
index++
}
if(points.size()!=coordinates.size()){
d.shape.coordinates = [points]
def udata = new BasicDBObject()
udata.shape = d.shape
community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}else{
println d._id
def coordinateArrs = d.shape.coordinates
def multiPoints = []
def updateFlag = false
for(def typeCoordinates : coordinateArrs){
def pointArrs = []
for(def coordinates : typeCoordinates){
def points = []
def index = 0
for(def arrs : coordinates){
if(index!=0 && index+1!= coordinates.size()){
if(!points.contains(arrs)){
points << arrs
}else{
println d._id +" 重复数据:"+arrs
updateFlag = true
}
}else{
points << arrs
}
index++
}
pointArrs << points
}
multiPoints << pointArrs
}
if(updateFlag){
d.shape.coordinates = multiPoints
def udata = new BasicDBObject()
udata.shape = d.shape
community_CONN.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
}
}
@@ -93,12 +93,23 @@ class DevProcess {
} }
def aiDevTown(){ def aiDevTown(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d -> ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\河北省开发区-ai结果.xlsx", { d ->
def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出") // def res = FangjiaClientUtils.aiResult("河北省哪些乡镇街道完全隶属于"+d['E']+",有则以逗号隔开方式输出,无则输出无,其它信息不输出")
def message = res.data.replaceAll("\n", "") // def message = res.data.replaceAll("\n", "")
d['F'] = res.code // d['F'] = res.code
d['G'] = message // d['G'] = message
println d['E'] +" "+res.code + " "+message // println d['E'] +" "+res.code + " "+message
def q = new BasicDBObject("name", d['E']).append("type","乡镇街道")
def dbcursor = DEV_MAPPING_CONN.find(q)
if(dbcursor.count()>0){
def list = []
for(def db : dbcursor){
list << db.township
}
d['H']= list.join(",")
}
}) })
} }
@@ -0,0 +1,75 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.looplineinfo.InPolygon
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.awt.geom.Point2D
class DistrictLoopInfo {
DBCollection LATLNG_POLYGON_COLL = MongoUtils.getCollectionByName("fangjia", "latlng_polygon", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictLoopInfo()
task.run()
}
def run(){
def q = new BasicDBObject()
q.put("name", new BasicDBObject('$in', ["内环","中环","外环","郊环"]))
def loopShapes = LATLNG_POLYGON_COLL.find(q)
def loopMap = [:]
for(def loop : loopShapes){
loopMap.put(loop.name, loop)
}
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city", "region", "name", "lat", "lng", "loop_line_info"])
println districts.size()
def index = 0
for(def d : districts){
index++
if(d.lng){
def lng = d.lng/10000000000.0
def lat = d.lat/10000000000.0
Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6, lng), DataHandleUtils.getDoubleRound(6, lat));
def loop1 = loopMap.get("内环")
def flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
def loop_line_info = ""
if(flag){
loop_line_info = "内环内"
}else{
loop1 = loopMap.get("中环")
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
if(flag){
loop_line_info = "内中环"
}else{
loop1 = loopMap.get("外环")
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
if(flag){
loop_line_info = "中外环"
}else{
loop1 = loopMap.get("郊环")
flag = InPolygon.isInArea(point, InPolygon.createPoint(loop1.shape.coordinates))
if(flag){
loop_line_info = "外郊环"
}else{
loop_line_info = "郊环外"
}
}
}
}
println index +" "+d.region +" "+d.name+" 原:"+d.loop_line_info+" 现:"+loop_line_info+" lng:"+lng+","+lat
def udata =new BasicDBObject()
udata.put("loop_line_info", loop_line_info)
DBUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set',udata))
}else{
println index + " "+d.region +" "+d.name+"无经纬度信息"
}
}
}
}
@@ -22,7 +22,47 @@ class DistrictRepeatBatch {
static main(args) { static main(args) {
def task = new DistrictRepeatBatch() def task = new DistrictRepeatBatch()
// task.run() // task.run()
task.resultProcess() // task.resultProcess()
task.processRepeatAndAlias()
}
def processRepeatAndAlias(){
ExcelUtils.modify("D:\\数据\\小区\\重复小区\\结果\\重复可疑小区-南京.xlsx",{d->
if(d['H'] && d['H'] == "删除"){
def obj = CommonDatautils.findDistrictById(d['A'])
if(!obj){
obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
}
if(obj){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
saveOptLog(obj.city, obj.region, obj.name, "remove")
}
}else{
if(d['I']){
def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(obj){
def arrs = d['I'].split(",")
if(arrs){
def alias = obj.alias
if(!alias){
alias = []
}
alias.addAll(arrs)
alias = alias.unique()
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
saveOptLog(obj.city, obj.region, obj.name, "update")
}
}
}
}
},0)
} }
def run(){ def run(){
@@ -17,7 +17,7 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
class GXDdistrictMatch { class 8GXDdistrictMatch {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -29,8 +29,8 @@ class TownMiddleProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL) DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL) DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping_0526", MONGO100_URL)
DBCollection TEMP_ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
@@ -65,7 +65,7 @@ class TownMiddleProcess {
//修改乡镇街道别名 //修改乡镇街道别名
// updateTownAlias() // updateTownAlias()
//中间表同步到100 //中间表同步到100
synchTown() // synchTown()
//添加地级市标志 //添加地级市标志
// addCityFlag() // addCityFlag()
// addPinyin() // addPinyin()
@@ -101,11 +101,22 @@ class TownMiddleProcess {
// updateNewRegionPoyline()//更新新的区域围栏 // updateNewRegionPoyline()//更新新的区域围栏
updateMappingPolyline() // updateMappingPolyline()
//批量修改多边型围栏数据结构
// updatePolylineBatch()
// test2dshape()
//将省会城市更新到数据库中
// updateProvinceCenter()
//城市级别更新到库中
// updateCityLevel()
//城市顺序
// updateCityNum()
} }
def createProvince(){ def createProvince(){
def cat = "省" def cat = "省"
def tjjMaps = findAllProcincesFromTJJ(cat) def tjjMaps = findAllProcincesFromTJJ(cat)
@@ -457,16 +468,17 @@ class TownMiddleProcess {
def datas = [] def datas = []
for(def d : dbcursor){ for(def d : dbcursor){
datas << d datas << d
if (datas.size() >= 2000) { if (datas.size() >= 5000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) // ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
// ADMINISTRATIVE_MAPPING_COLL.insert(datas)
println datas.size() println datas.size()
datas.clear(); datas.clear();
} }
} }
if (datas.size() >= 0) { if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); ADMINISTRATIVE_MAPPING_COLL.insert(datas)
// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) // ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear(); datas.clear();
} }
@@ -1449,4 +1461,123 @@ class TownMiddleProcess {
} }
def updatePolylineBatch(){
def q = new BasicDBObject()
q.put("shape",new BasicDBObject('$exists', true))
q.put("update_shape",new BasicDBObject('$exists', false))
def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100)
for(def d : dbcursor){
def type = d.shape.type
def udata = new BasicDBObject()
def shapeMap = [:]
if(type == "polygon"){
shapeMap.put("type", "Polygon")
shapeMap.put("coordinates", d.shape.coordinates)
udata.put("shape", shapeMap)
udata.put("update_shape", 1)
ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata))
}else if(type == "MultiPolygon"){
def coordinates = d.shape.coordinates
def new_coors = []
for(def coors : coordinates){
new_coors << [coors]
}
shapeMap.put("type", "MultiPolygon")
shapeMap.put("coordinates", new_coors)
udata.put("shape", shapeMap)
udata.put("update_shape", 1)
ADMINISTRATIVE_MAPPING_COLL_100.update(new BasicDBObject('_id', d._id), new BasicDBObject('$set', udata))
}
}
}
def test2dshape(){
def q = new BasicDBObject()
q.put("street","花亭路街道")
// q.put("shape",new BasicDBObject('$exists', true))
def dbcursor = ADMINISTRATIVE_MAPPING_COLL_100.find(q).batchSize(100)
for(def obj : dbcursor){
try{
TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj)
}catch(Exception e){
def coordinates = obj.shape.coordinates
for(def coors : coordinates){
for (int i = 0; i < coors.size(); i++) { // 改为索引遍历
def coorsArr = coors[i]
def index = 0
def new_coors = []
for(def points : coorsArr){
index ++
if(!new_coors.contains(points)){
new_coors << points
}else{
if(index<=coorsArr.size()){
new_coors << points
break;
}
}
}
if(new_coors.size()!=coorsArr.size()){
coors[i] = new_coors
println "[修改] "+ obj.city +" "+obj.street +" 原:"+coorsArr.size()+" 现:"+new_coors.size()
}
}
}
try{
TEMP_ADMINISTRATIVE_MAPPING_COLL_100.save(obj)
}catch(Exception ex){
println "仍有问题:"+obj.city +" "+obj.street+" "+ex.getMessage()
}
}
}
}
def updateProvinceCenter(){
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\省会城市.xlsx",{d->
def city = d['C']
if(city){
def q = new BasicDBObject("type","城市").append("city_short", city)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject("capital_city", "是")
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
}
}
})
}
def updateCityLevel(){
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市等级.xlsx",{d->
def city = d['A']
if(city){
def q = new BasicDBObject("type","城市").append("city_short", city)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject("city_level", d['B'])
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
}else{
println city +" 没找到"
}
}
})
}
def updateCityNum(){
ExcelUtils.modify("D:\\数据\\数据建设\\城市\\城市排序.xlsx",{d->
def city = d['A']
if(city){
def q = new BasicDBObject("type","城市").append("city_short", city)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject("city_position", d['B'].toString().toDouble().toInteger())
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', udata))
}else{
println city +" 没找到"
}
}
})
}
} }
@@ -18,16 +18,17 @@ class GaodeCityRange {
} }
def run(){ def run(){
def citys = CommonDatautils.findAllCity() // def citys = CommonDatautils.findAllCity()
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"] def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
def citys = ["新星","胡杨河","白杨"]
HttpRequest request = new HttpRequest() HttpRequest request = new HttpRequest()
def rows = [] def rows = []
citys.each{c-> citys.each{c->
if(!citys1.contains(c)){ if(!citys1.contains(c)){
c = c.replace("州","") c = c.replace("州","")
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=66e6771767ab293ace441553c36621c1&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
if(resp.containsKey("districts")){ if(resp.containsKey("districts")){
def datas = resp.get("districts") def datas = resp.get("districts")
for(def d : datas){ for(def d : datas){
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.location
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.Bytes import com.mongodb.Bytes
@@ -22,6 +23,11 @@ class RegionPointCrawl {
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static DBCollection TOWN_CENTERCOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_center")
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) { static main(args) {
def task = new RegionPointCrawl(); def task = new RegionPointCrawl();
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省 //23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
@@ -40,8 +46,12 @@ class RegionPointCrawl {
// println "区域围栏 完成" // println "区域围栏 完成"
// } // }
task.statSame() // task.statSame()
//"福建省","河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","山东省","湖北省","湖南省","江西省","江苏省","安徽省"
def provinces = ["河北省","山西省","吉林省","辽宁省","黑龙江省","陕西省","甘肃省","青海省","山东省","福建省","浙江省","台湾省","河南省","湖北省","湖南省","江西省","江苏省","安徽省","广东省","海南省","四川省","贵州省","云南省","北京市","上海市","天津市","重庆市","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
task.getTownshipCenter(provinces)
task.addTownCenter(provinces)
} }
def crawlCityAndDistrictCode(def province){ def crawlCityAndDistrictCode(def province){
@@ -210,7 +220,7 @@ class RegionPointCrawl {
jo = JSONObject.fromObject(html) jo = JSONObject.fromObject(html)
if(jo.infocode!="10000"){ if(jo.infocode!="10000"){
errorCount ++ errorCount ++
Thread.sleep(30*1000) Thread.sleep(3*1000)
// println " 不可用Key: "+key // println " 不可用Key: "+key
}else{ }else{
errorCount = 100 errorCount = 100
@@ -228,15 +238,15 @@ class RegionPointCrawl {
static String getRandomKey2() { static String getRandomKey2() {
def KeyList = [ def KeyList = [
"2af0b8533415a612b98d59bb93d664b9", // "2af0b8533415a612b98d59bb93d664b9",
"0246c57bbcf24aa246c207b3c99dfe41", // "0246c57bbcf24aa246c207b3c99dfe41",
"042328b3f0fa5daf426802d379521d87", // "042328b3f0fa5daf426802d379521d87",
"055d4f1b5b64a2bce18f4fe1f15c2639", // "055d4f1b5b64a2bce18f4fe1f15c2639",
"d5484f5170502405d88bbaffe1b496f0", // "d5484f5170502405d88bbaffe1b496f0",
"e74ccc4258079d2200199695110cceb1", // "e74ccc4258079d2200199695110cceb1",
"f58629fc799fbc8537ca85046c6d44a5", // "f58629fc799fbc8537ca85046c6d44a5",
"291073f17ee0b963ccb1927ed92bf265", "eb69e4ad2a9cdd31f163a0ed32effe6e"
"2a1fb75990f2b39f6221e50892dd3988" // "2af0b8533415a612b98d59bb93d664b9"
] ]
def count = RandomUtils.nextInt(KeyList.size()) def count = RandomUtils.nextInt(KeyList.size())
@@ -300,4 +310,117 @@ class RegionPointCrawl {
} }
return map; return map;
} }
def getTownshipCenter(provinces){
for(def p : provinces){
def query = new BasicDBObject("province", p)
def count = TOWN_CENTERCOLL.count(query)
if(count){
println p +" 已存在"
}else{
def ret = polygon(p, "3", "base");
if(ret){
def citys = ret[0].districts
if(citys){
def allRets = []
for(def c : citys){
def cityTowns = getTownByCity(p, c.name,c.districts)
allRets.addAll(cityTowns)
}
saveTownCenter(allRets)
println p +" 保存:"+allRets.size()
}
}else{
println p+ "无结果"
}
}
}
}
def getTownByCity(prov, city,distDatas){
def rets = []
for(def d : distDatas){
def townDatas = d.districts
for(def town : townDatas){
town.province = prov
town.district = d.name
town.city = city
rets << town
}
}
return rets
}
def saveTownCenter(allRets){
for(def r : allRets){
def obj = new BasicDBObject()
obj.put("province", r.province)
obj.put("city", r.city)
obj.put("district", r.district)
obj.put("town", r.name)
obj.put("center", r.center)
obj.put("level", r.level)
TOWN_CENTERCOLL.save(obj)
}
}
def addTownCenter(provinces){
for(def p : provinces){
def q = new BasicDBObject("province",p).append("type","乡镇街道")
// q.put("location", new BasicDBObject('$exists', false))
def dbcursor = administrative_mapping_coll.find(q)
for(def obj : dbcursor){
def tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", obj.street)
def dbo = TOWN_CENTERCOLL.findOne(tq)
if(!dbo){
def new_street = obj.street.replace("管理处","")
tq = new BasicDBObject("city", obj.city).append("district", obj.mzb_district_name).append("town", new_street)
dbo = TOWN_CENTERCOLL.findOne(tq)
}
if(dbo){
def arrs = dbo.center.split(",")
def location = [:]
location["lon"] = arrs[0].toDouble()
location["lat"] = arrs[1].toDouble()
if(!obj.location){
def udata = new BasicDBObject("location", location)
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
println("【修改】"+obj.city +" "+ obj.district+" "+obj.street)
}else{
def distance = com.fangjia.script.util.GeoUtils.calculateDistance(location.lat, location.lon, obj.location.lat, obj.location.lon);
if(distance>1000){
def udata = new BasicDBObject("location", location)
println(obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 经纬度中心点相距1公里以外 原:"+obj.location.lon+","+obj.location.lat+" 新:"+dbo.center+" 距离:"+distance)
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
}
}else{
def esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street)
if(!esobj){
def short_prov = obj.province.replace("省", "")
if(obj.street.contains(short_prov)){
esobj = CommonDatautils.findGaoDeDatas2(obj.city, obj.mzb_district_name, obj.street.replace(short_prov, ""))
}
}
if(esobj){
def location = [:]
location["lon"] = esobj.location.lon.toDouble()
location["lat"] = esobj.location.lat.toDouble()
def udata = new BasicDBObject("location", location)
administrative_mapping_coll.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
println("【修改】"+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street+" "+ esobj.cat+" "+esobj.name +" es poi ")
}else{
println(obj.province+" "+obj.city +" "+ obj.district+" "+obj.mzb_district_name+" "+obj.street +" 没有找到")
}
}
}
}
}
} }
@@ -0,0 +1,15 @@
package com.fangjia.script.gj.test
class NonghangDataExport {
static main(args) {
def task = new NonghangDataExport();
task.run()
}
def run(){
def citys = ["ÁÙÒÊ","¹ãÖÝ", "ÉîÛÚ"]
for(def c : citys){
}
}
}
@@ -2,6 +2,7 @@ package com.fangjia.script.linxsh.util
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBCursor import com.mongodb.DBCursor
@@ -47,6 +48,8 @@ class AddressUtil {
private static final cityMap = new HashMap<String, String>() private static final cityMap = new HashMap<String, String>()
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
static { static {
initCityMap() initCityMap()
} }
@@ -55,9 +58,7 @@ class AddressUtil {
* 初始化城市数据 * 初始化城市数据
*/ */
static def initCityMap() { static def initCityMap() {
Mongo mongo = new Mongo("mongo.db.fangjia.com", 28017); DBCursor cursor = SERVER_AREA_COLL.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1));
DBCollection SERVER_AREA = mongo.getDB("fangjia").getCollection("server_area");
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1));
for(DBObject dbObject : cursor) { for(DBObject dbObject : cursor) {
if(!"全国".equals(dbObject.display)) { if(!"全国".equals(dbObject.display)) {
cityMap.put(dbObject.display, dbObject.display) cityMap.put(dbObject.display, dbObject.display)
@@ -239,7 +239,7 @@ public class CommonUtils {
println(cmd) println(cmd)
DBObject data = new BasicDBObject() DBObject data = new BasicDBObject()
data.put("category", "sell") data.put("category", "sell")
data.put("priority", 3) data.put("priority", 1)
data.put("status", 0) data.put("status", 0)
data.put("need_redraw_trend", needRedrawTrend) data.put("need_redraw_trend", needRedrawTrend)
data.put("remark", 'week') data.put("remark", 'week')
@@ -1548,6 +1548,34 @@ public class OpenApiClient {
} }
public static JSONObject houseAction(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/house/around", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/house/around", params, "utf-8")
println("${city} ${address} ³ÌÐòµ÷ÓÃÒì³£")
Thread.sleep(6000)
}
if (result.getInt("code") == 0) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
} }
@@ -21,15 +21,15 @@ public class DistrictMonthlyCalc{
public static void main(String[] args) { public static void main(String[] args) {
List<String> cities = ["ÉϺ£"]; List<String> cities = ["ÏåÑô"];
def allCity = CommonUtils.findAllCity() // def allCity = CommonUtils.findAllCity()
cities.addAll(allCity) // cities.addAll(allCity)
def districts = [] def districts = []
cities.each {def city-> cities.each {def city->
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("name", 1).append("region", 1)).batchSize(2000).toList() districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("name", 1).append("region", 1)).batchSize(2000).toList()
println(city) println(city)
} }
ExecutorService executorService = Executors.newFixedThreadPool(20) ExecutorService executorService = Executors.newFixedThreadPool(1)
districts.each {BasicDBObject district-> districts.each {BasicDBObject district->
executorService.submit(new Calculator(district)) executorService.submit(new Calculator(district))
} }
@@ -43,7 +43,7 @@ public class DistrictMonthlyCalc{
class Calculator extends Thread{ class Calculator extends Thread{
static DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static DBCollection REGION_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") static DBCollection REGION_STAT_TREND = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat_new", "mongo.db.fangjia.com")
final BasicDBObject district; final BasicDBObject district;
public Calculator(final BasicDBObject district){ public Calculator(final BasicDBObject district){
@@ -52,14 +52,22 @@ class Calculator extends Thread{
@Override @Override
void run() { void run() {
REGION_STAT.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo-> def q = new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)
q.put("weekend", new BasicDBObject('$gte', DateUtil.parseDate("2022-01-01", "yyyy-MM-dd")))
REGION_STAT.find(q).sort(new BasicDBObject("weekend", 1).append("m_date",1)).each {BasicDBObject dbo->
Date weekend = dbo.weekend Date weekend = dbo.weekend
if(DateUtil.isLastSundayOfMonth(weekend)){ if(DateUtil.isLastSundayOfMonth(weekend)){
def trend = new BasicDBObject("city", district.city) def trend = new BasicDBObject("city", district.city)
trend.append("region", district.region) trend.append("region", district.region)
trend.append("district_name", district.name) trend.append("district_name", district.name)
trend.append("date", weekend.format("yyyyMM").toInteger()) trend.append("date", weekend.format("yyyyMM").toInteger())
REGION_STAT_TREND.update(trend , new BasicDBObject('$set', new BasicDBObject("avg_price", dbo.avg_price)),true,false) def udata = new BasicDBObject()
if(dbo.avg_price_2b && dbo.value_type!=1){
udata.put("avg_price", dbo.avg_price_2b)
}else{
udata.put("avg_price", dbo.avg_price)
}
REGION_STAT_TREND.update(trend , new BasicDBObject('$set', udata),true,false)
} }
} }
REGION_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("data", 1)).each {BasicDBObject trend-> REGION_STAT_TREND.find(new BasicDBObject("city", district.city).append("region", district.region).append("district_name", district.name)).sort(new BasicDBObject("data", 1)).each {BasicDBObject trend->