update
This commit is contained in:
+4
-2
@@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
// def endDate = 202208
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
@@ -71,7 +71,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
@@ -87,7 +89,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println city +" 成交共:"+rets.size()
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
|
||||
+18
@@ -68,6 +68,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
def index = 0
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
statCity(citys)
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
@@ -82,6 +83,23 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def statCity(def citys){
|
||||
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1)
|
||||
def houseCitys = []
|
||||
for(def d : datas){
|
||||
houseCitys << d['C']
|
||||
}
|
||||
for(def city : citys){
|
||||
if(!houseCitys.contains(city)){
|
||||
def scity = CcbCommonUtils.specialCityMap.get(city)
|
||||
if(!scity){
|
||||
println city
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
+27
-26
@@ -31,39 +31,40 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
|
||||
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
|
||||
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
|
||||
def citys = CcbCommonUtils.citys_252
|
||||
|
||||
|
||||
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_丧葬设施_252.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_医院_聚类2_252.xlsx", 0,1)
|
||||
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_地铁_252.xlsx", 0,1)
|
||||
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_公交_252.xlsx", 0,1)
|
||||
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_银行_252.xlsx", 0,1)
|
||||
|
||||
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_学校_252.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
|
||||
def rets = []
|
||||
|
||||
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// rets.addAll(rows1)
|
||||
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
rets.addAll(rows1)
|
||||
|
||||
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
rets.addAll(rows2)
|
||||
//
|
||||
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
// rets.addAll(rows2)
|
||||
//
|
||||
// def rows3 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
// rets.addAll(rows3)
|
||||
|
||||
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
rets.addAll(rows4)
|
||||
|
||||
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
rets.addAll(rows5)
|
||||
////
|
||||
// def rows4 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
// rets.addAll(rows4)
|
||||
//
|
||||
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||
// rets.addAll(rows6)
|
||||
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||
rets.addAll(rows6)
|
||||
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
|
||||
|
||||
// def rows5 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
// rets.addAll(rows5)
|
||||
//
|
||||
|
||||
def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
@@ -260,9 +261,9 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
}else if(type =="丧葬服务"){
|
||||
ret << "ob101"
|
||||
}else if(type =="学校"){
|
||||
if(d['F'].contains("中学")){
|
||||
if(d['B'].contains("中学")){
|
||||
ret << "ed138"
|
||||
}else if(d['F'].contains("小学")){
|
||||
}else if(d['B'].contains("小学")){
|
||||
ret << "ed114"
|
||||
}
|
||||
}
|
||||
|
||||
+1
@@ -417,6 +417,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","云山尚城")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
|
||||
+4
-5
@@ -370,13 +370,12 @@ class DealPriceDataProcess {
|
||||
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def month = 202208
|
||||
def citys = CcbCommonUtils.citys_92
|
||||
def month = 202209
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202209)
|
||||
def rets = findPeGteThirdParty(city, 202210)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
for(def m : rets){
|
||||
@@ -388,7 +387,7 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
|
||||
def rets = []
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
|
||||
+1
@@ -171,6 +171,7 @@ class ExportOracleData {
|
||||
|
||||
def exportCommunityStaging() {
|
||||
|
||||
def allRows = []
|
||||
def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj";
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
|
||||
+2
-2
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.citys_252;
|
||||
def rows = []
|
||||
|
||||
for (def city : citys) {
|
||||
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-09-22", "2022-09-23")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+6
-6
@@ -22,10 +22,10 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202206
|
||||
def endDate = 202208
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202107
|
||||
def endDate = 202109
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
@@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.citys_252;
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
|
||||
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-08-01"
|
||||
static def statDate = "2022-09-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+12
-4
@@ -49,7 +49,15 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def citys = getAllCityUrls(driver)
|
||||
citys = citys.reverse()
|
||||
// citys = citys.reverse()
|
||||
// def index = 0
|
||||
// for(def c : citys){
|
||||
// index++
|
||||
// if(c.city=="合肥"){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// citys = citys[index..citys.size()-1]
|
||||
def start = 0
|
||||
// for(def i=0;i<citys.size();i++){
|
||||
// def m = citys[i]
|
||||
@@ -190,7 +198,7 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
@@ -390,13 +398,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("福州")){
|
||||
// if(e.text().contains("合肥")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
// cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+1
-1
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -38,13 +38,13 @@ class PaimaiDataSend {
|
||||
// task.jdRunByMongo()
|
||||
|
||||
// task.aliZcRun()
|
||||
// task.aliZcRun2()
|
||||
task.aliZcRun2()
|
||||
// task.aliFpRun()
|
||||
// task.rmfysszcRun()
|
||||
// task.jdRun()
|
||||
// task.checkUrl()
|
||||
|
||||
task.paimaiSendToEs()
|
||||
// task.paimaiSendToEs()
|
||||
}
|
||||
|
||||
|
||||
@@ -337,7 +337,7 @@ class PaimaiDataSend {
|
||||
q.put("province","江苏")
|
||||
q.put("sub_type","土地")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',2))
|
||||
// q.put("url","https://sf-item.taobao.com/sf_item/668496148492.htm")
|
||||
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
// q.put("parser_status", new BasicDBObject('$exists',false))
|
||||
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
||||
@@ -424,7 +424,7 @@ class PaimaiDataSend {
|
||||
q.put("sub_type","土地")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',1))
|
||||
// q.put("region","姑苏")
|
||||
// q.put("url","https://susong-item.taobao.com/auction/674736290116.htm")
|
||||
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
|
||||
def dbcursor = ali_zc_coll.find(q).batchSize(1000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
|
||||
Reference in New Issue
Block a user