Merge branch 'script_gj' into 'master'

新增导测试数据

See merge request !639
This commit is contained in:
2023-05-22 16:22:14 +08:00
9 changed files with 1062 additions and 15 deletions
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-20")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -92,13 +92,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-01-01"
def date = "2023-02-01"
def endDate = "2023-03-01"
def newDate = "2023-04-01"
def startDate = "2023-02-01"
def date = "2023-03-01"
def endDate = "2023-04-01"
def newDate = "2023-05-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -1,5 +1,7 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
@@ -11,7 +13,26 @@ class UpdateData {
static main(args) {
def task = new UpdateData();
task.run()
// task.run()
task.updateBlock()
}
def updateBlock(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "block","name"])
for(def d : dists){
if(d.block){
def newBlock = FiledStrand.processBlock(d.city, d.region, d.block)
if(newBlock!= d.block){
def udata = new BasicDBObject()
udata.put("block", newBlock)
println d.city+" "+d.block +" "+newBlock
DbUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
}
}
}
}
}
def run(){
@@ -128,9 +128,9 @@ class CityInfoAnaBatch extends BaseBatch {
SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); }
allCitys.each {k,v ->
// if(["南京"].contains(k.toString())){
if(["漳州"].contains(k.toString())){
CITYS << k.toString();
// }
}
}
// 所有城市的人口信息
@@ -1,5 +1,7 @@
package com.fangjia.script.gj
import com.fangjia.core.BlockData
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
@@ -337,11 +339,12 @@ class FiledStrand {
return null
}
static def processBlock(def block){
static def processBlock(city, region, def block){
if(block){
if(block.contains("无") || block.toString().length()>10){
return null
}
// block = BlockData.formatBlock(city, region, block);
return block
}
return null
@@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity().reverse()
def citys = ["漳州"]//CommonDatautils.findAllCity().reverse()
def analysRets = []
@@ -279,7 +279,7 @@ class DistrictFieldAddBatch extends BaseBatch{
newVal = FiledStrand.processAddress(esVal, dist.city)
}
if(seaweedfield =="block"){
newVal = FiledStrand.processBlock(esVal)
newVal = FiledStrand.processBlock(dist.city, dist.region, esVal)
}
if(seaweedfield == "build_type"){
@@ -26,8 +26,10 @@ class SeaweedDataMatchProcess {
rMap.region = obj.region
rMap.alias = obj.alias
rMap.name = obj.name
rMap.lat = obj.lat/10000000000.0
rMap.lng = obj.lng/10000000000.0
if(obj.lat){
rMap.lat = obj.lat/10000000000.0
rMap.lng = obj.lng/10000000000.0
}
rMap.id = obj._id.toString()
rMap.address = obj.address
}
@@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome {
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
//广州,佛山 重庆
static main(args) {
def task = new BeiKeCrawlByChrome();
task.run()
@@ -447,7 +448,7 @@ public class BeiKeCrawlByChrome {
btn.submit(); // 点击行为;
Thread.sleep(10000)
}catch(Exception e){
Thread.sleep(60000)
Thread.sleep(10000)
}
return driver
}