Merge branch 'script_gj' into 'master'

Script gj

See merge request !409
This commit is contained in:
2022-10-24 11:49:28 +08:00
11 changed files with 79 additions and 50 deletions
@@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{
// def endDate = 202208
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -71,7 +71,9 @@ class BigdataDealBatch extends BaseBatch{
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
@@ -87,7 +89,7 @@ class BigdataDealBatch extends BaseBatch{
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
@@ -68,6 +68,7 @@ class BigdataForsaleBatch extends BaseBatch{
public void process(Object args) {
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
statCity(citys)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
@@ -82,6 +83,23 @@ class BigdataForsaleBatch extends BaseBatch{
}
}
def statCity(def citys){
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1)
def houseCitys = []
for(def d : datas){
houseCitys << d['C']
}
for(def city : citys){
if(!houseCitys.contains(city)){
def scity = CcbCommonUtils.specialCityMap.get(city)
if(!scity){
println city
}
}
}
}
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def totalCount = new AtomicLong()
@@ -31,39 +31,40 @@ class BigdataPoiBatch extends BaseBatch{
@Override
public void process(Object args) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def citys = CcbCommonUtils.citys_252
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_丧葬设施_252.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_医院_聚类2_252.xlsx", 0,1)
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_地铁_252.xlsx", 0,1)
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_公交_252.xlsx", 0,1)
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_银行_252.xlsx", 0,1)
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_学校_252.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
def rets = []
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows1)
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
// rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
// rets.addAll(rows3)
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
////
// def rows4 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
// rets.addAll(rows4)
//
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
// rets.addAll(rows6)
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
// def rows5 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
// rets.addAll(rows5)
//
def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
if(CommonDatautils.isWindow()){
@@ -260,9 +261,9 @@ class BigdataPoiBatch extends BaseBatch{
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
if(d['F'].contains("中学")){
if(d['B'].contains("中学")){
ret << "ed138"
}else if(d['F'].contains("小学")){
}else if(d['B'].contains("小学")){
ret << "ed114"
}
}
@@ -417,6 +417,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","云山尚城")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -370,13 +370,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def month = 202208
def citys = CcbCommonUtils.citys_92
def month = 202209
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202209)
def rets = findPeGteThirdParty(city, 202210)
def total = 0
println city +" "+rets.size()
for(def m : rets){
@@ -388,7 +387,7 @@ class DealPriceDataProcess {
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys_252;
def rows = []
for (def city : citys) {
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-09-22", "2022-09-23")
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -22,10 +22,10 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202207
def endDate = 202209
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202107
def endDate = 202109
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
}
@@ -52,7 +52,7 @@ class YouDaDataParser {
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
// task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -49,7 +49,15 @@ public class BeiKeCrawlByChrome {
WebDriver driver = new ChromeDriver()
def citys = getAllCityUrls(driver)
citys = citys.reverse()
// citys = citys.reverse()
// def index = 0
// for(def c : citys){
// index++
// if(c.city=="合肥"){
// break
// }
// }
// citys = citys[index..citys.size()-1]
def start = 0
// for(def i=0;i<citys.size();i++){
// def m = citys[i]
@@ -190,7 +198,7 @@ public class BeiKeCrawlByChrome {
}
page++;
}else{
stop = true;
stop = true;
}
}
return detailUrls
@@ -390,13 +398,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("福州")){
// if(e.text().contains("合肥")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
// cityMap.put(e.text(), "https:"+href)
}
// }
}
}
return list
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
@@ -38,13 +38,13 @@ class PaimaiDataSend {
// task.jdRunByMongo()
// task.aliZcRun()
// task.aliZcRun2()
task.aliZcRun2()
// task.aliFpRun()
// task.rmfysszcRun()
// task.jdRun()
// task.checkUrl()
task.paimaiSendToEs()
// task.paimaiSendToEs()
}
@@ -337,7 +337,7 @@ class PaimaiDataSend {
q.put("province","江苏")
q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',2))
// q.put("url","https://sf-item.taobao.com/sf_item/668496148492.htm")
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
//https://sf-item.taobao.com/sf_item/657131607486.htm
// q.put("parser_status", new BasicDBObject('$exists',false))
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
@@ -424,7 +424,7 @@ class PaimaiDataSend {
q.put("sub_type","土地")
// q.put("parser_status", new BasicDBObject('$ne',1))
// q.put("region","姑苏")
// q.put("url","https://susong-item.taobao.com/auction/674736290116.htm")
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
def dbcursor = ali_zc_coll.find(q).batchSize(1000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){