update
This commit is contained in:
+10
-8
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
|||||||
/**
|
/**
|
||||||
* 深圳公众号成交数据录入
|
* 深圳公众号成交数据录入
|
||||||
*/
|
*/
|
||||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
|
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1)
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
|
||||||
@@ -72,19 +72,21 @@ for(def d : datas){
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
map.area = d['F'].toString().toDouble()
|
map.area = d['F'].replace("平","").toString().toDouble()
|
||||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000)
|
||||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble())
|
||||||
if(d['I']){
|
if(d['I']){
|
||||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
map.trade_date = d['J']
|
||||||
// def trade_date = d['J']
|
// def trade_date = d['J']
|
||||||
// if(trade_date){
|
// if(trade_date){
|
||||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||||
// }else{
|
// }
|
||||||
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
// else{
|
||||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||||
|
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||||
// }
|
// }
|
||||||
|
|
||||||
map.source = "深房家族"
|
map.source = "深房家族"
|
||||||
|
|||||||
+15
-9
@@ -35,7 +35,7 @@ import java.util.regex.Pattern
|
|||||||
@BatchOwner(Watcher.GaoJuan)
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
class BigdataDealBatch extends BaseBatch{
|
class BigdataDealBatch extends BaseBatch{
|
||||||
|
|
||||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
|
|
||||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||||
@@ -726,18 +726,24 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
t.name = t.name +"商住楼"
|
t.name = t.name +"商住楼"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
def flag = false
|
||||||
if(!rMap){
|
def address = null
|
||||||
|
if(t.address){
|
||||||
}
|
flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||||
if(!rMap && t.address){
|
|
||||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
|
||||||
if(flag){
|
if(flag){
|
||||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
address = t.address
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address);
|
||||||
|
if(!rMap && !t.name.toString().endsWith("小区")){
|
||||||
|
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address);
|
||||||
|
}
|
||||||
|
|
||||||
if(rMap && rMap.region==region){
|
if(rMap && rMap.region==region){
|
||||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||||
|
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
|
||||||
|
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||||
}else {
|
}else {
|
||||||
if (rMap) {
|
if (rMap) {
|
||||||
println "区域不同"
|
println "区域不同"
|
||||||
@@ -755,7 +761,7 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
)
|
||||||
}
|
}
|
||||||
latch.await()
|
latch.await()
|
||||||
println city +" "+trades.size()+" "+new Date().toString()
|
println city +" "+trades.size()+" "+new Date().toString()
|
||||||
|
|||||||
+2
-1
@@ -634,7 +634,8 @@ class CcbCommonUtils {
|
|||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||||
q.put("match_id", new BasicDBObject('$exists', false))
|
q.put("match_id", new BasicDBObject('$exists', false))
|
||||||
// q.put("district_name","中海寰宇天下")
|
// q.put("district_name","新明胡同")
|
||||||
|
// q.put("address","桂苑大道135号")
|
||||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||||
def total = dbcursor.count()
|
def total = dbcursor.count()
|
||||||
def rets = new HashSet()
|
def rets = new HashSet()
|
||||||
|
|||||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
|||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
|
|
||||||
//添加自己库中的人工指导价
|
//添加自己库中的人工指导价
|
||||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08")
|
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-21")
|
||||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||||
if(prices){
|
if(prices){
|
||||||
def distMap = getDistrictMap(city)
|
def distMap = getDistrictMap(city)
|
||||||
|
|||||||
+4
-4
@@ -40,8 +40,8 @@ class TradeStatLog {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new TradeStatLog()
|
def task = new TradeStatLog()
|
||||||
task.run()//统计最近3个月抓取的成交数据量
|
// task.run()//统计最近3个月抓取的成交数据量
|
||||||
// task.exportTrade()//导出未入库成交
|
task.exportTrade()//导出未入库成交
|
||||||
// task.thirdPartyStat()
|
// task.thirdPartyStat()
|
||||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||||
// task.averageStat()
|
// task.averageStat()
|
||||||
@@ -499,8 +499,8 @@ class TradeStatLog {
|
|||||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||||
def rets = []
|
def rets = []
|
||||||
rets << headers
|
rets << headers
|
||||||
def startDate = "2023-08-01"
|
def startDate = "2024-04-01"
|
||||||
def endDate = "2023-10-01"
|
def endDate = "2024-06-01"
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
|
|
||||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||||
|
|||||||
+1
-1
@@ -4679,7 +4679,7 @@ class CommonDatautils {
|
|||||||
|
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("type","乡镇街道")
|
q.put("type","乡镇街道")
|
||||||
q.put("province",new BasicDBObject('$in', ['福建省','重庆市']))
|
q.put("province",new BasicDBObject('$in', ['浙江省','江苏省']))
|
||||||
|
|
||||||
def keys = new BasicDBObject()
|
def keys = new BasicDBObject()
|
||||||
keys.put("province",1)
|
keys.put("province",1)
|
||||||
|
|||||||
@@ -13,12 +13,15 @@ import net.sf.json.groovy.JsonSlurper
|
|||||||
import org.apache.commons.lang.StringUtils
|
import org.apache.commons.lang.StringUtils
|
||||||
|
|
||||||
class OpenApiUtils {
|
class OpenApiUtils {
|
||||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){
|
||||||
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
||||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||||
if(preciseLevelflag){
|
if(preciseLevelflag){
|
||||||
url = url +"&preciseLevel=strict"
|
url = url +"&preciseLevel=strict"
|
||||||
}
|
}
|
||||||
|
if(suggestName){
|
||||||
|
url = url +"&suggestName="+suggestName
|
||||||
|
}
|
||||||
String content = new HttpRequest().get(url, "utf-8");
|
String content = new HttpRequest().get(url, "utf-8");
|
||||||
def rMap = [:]
|
def rMap = [:]
|
||||||
if(content){
|
if(content){
|
||||||
|
|||||||
+5
-7
@@ -36,18 +36,16 @@ class SeaweedDataMatchProcess {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!rMap && name){
|
if(!rMap && address && name){
|
||||||
rMap = OpenApiUtils.matchDistrict(city, name, region, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
if(!rMap && address){
|
|
||||||
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||||
if(flag){
|
if(flag){
|
||||||
rMap = OpenApiUtils.matchDistrict(city, address, region, true)
|
rMap = OpenApiUtils.matchDistrict(city, address, region, true, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(!rMap && name){
|
||||||
|
rMap = OpenApiUtils.matchDistrict(city, name, region, false, null)
|
||||||
|
}
|
||||||
// if(rMap && rMap.id && region.contains(rMap.region)) {
|
// if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||||
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
|
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
|
||||||
// if (obj) {
|
// if (obj) {
|
||||||
|
|||||||
Reference in New Issue
Block a user