Merge branch 'script_gj' into 'master'
Script gj See merge request !409
This commit is contained in:
+4
-2
@@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
// def endDate = 202208
|
// def endDate = 202208
|
||||||
|
|
||||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||||
|
|
||||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||||
@@ -71,7 +71,9 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
def rows = []
|
def rows = []
|
||||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
|
def index = 0
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
|
index ++
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
def ppeDists = [:]
|
def ppeDists = [:]
|
||||||
|
|
||||||
@@ -87,7 +89,7 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
def trades = getCityData(city, statMap, ppeDists)
|
def trades = getCityData(city, statMap, ppeDists)
|
||||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||||
rows.addAll(rets)
|
rows.addAll(rets)
|
||||||
println city +" 成交共:"+rets.size()
|
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||||
|
|
||||||
def cityDatas = []
|
def cityDatas = []
|
||||||
cityDatas << headers
|
cityDatas << headers
|
||||||
|
|||||||
+18
@@ -68,6 +68,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def index = 0
|
def index = 0
|
||||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||||
|
statCity(citys)
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
index++
|
index++
|
||||||
@@ -82,6 +83,23 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def statCity(def citys){
|
||||||
|
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1)
|
||||||
|
def houseCitys = []
|
||||||
|
for(def d : datas){
|
||||||
|
houseCitys << d['C']
|
||||||
|
}
|
||||||
|
for(def city : citys){
|
||||||
|
if(!houseCitys.contains(city)){
|
||||||
|
def scity = CcbCommonUtils.specialCityMap.get(city)
|
||||||
|
if(!scity){
|
||||||
|
println city
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
|
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
|
||||||
|
|
||||||
def totalCount = new AtomicLong()
|
def totalCount = new AtomicLong()
|
||||||
|
|||||||
+27
-26
@@ -31,39 +31,40 @@ class BigdataPoiBatch extends BaseBatch{
|
|||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
def citys = CcbCommonUtils.citys;
|
def citys = CcbCommonUtils.citys_252
|
||||||
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
|
|
||||||
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
|
||||||
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_丧葬设施_252.xlsx", 0,1)
|
||||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
|
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_医院_聚类2_252.xlsx", 0,1)
|
||||||
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_地铁_252.xlsx", 0,1)
|
||||||
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_公交_252.xlsx", 0,1)
|
||||||
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
|
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_银行_252.xlsx", 0,1)
|
||||||
|
|
||||||
|
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_学校_252.xlsx", 0,1)
|
||||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
|
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
|
||||||
def rets = []
|
def rets = []
|
||||||
|
|
||||||
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||||
// rets.addAll(rows1)
|
// rets.addAll(rows1)
|
||||||
|
|
||||||
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
|
||||||
rets.addAll(rows1)
|
|
||||||
|
|
||||||
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
|
||||||
rets.addAll(rows2)
|
|
||||||
//
|
//
|
||||||
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||||
|
// rets.addAll(rows2)
|
||||||
|
//
|
||||||
|
// def rows3 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||||
// rets.addAll(rows3)
|
// rets.addAll(rows3)
|
||||||
|
////
|
||||||
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
// def rows4 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||||
rets.addAll(rows4)
|
// rets.addAll(rows4)
|
||||||
|
|
||||||
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
|
||||||
rets.addAll(rows5)
|
|
||||||
//
|
//
|
||||||
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||||
// rets.addAll(rows6)
|
rets.addAll(rows6)
|
||||||
|
|
||||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
|
|
||||||
|
// def rows5 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||||
|
// rets.addAll(rows5)
|
||||||
|
//
|
||||||
|
|
||||||
|
def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
|
||||||
|
|
||||||
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
|
def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat"
|
||||||
if(CommonDatautils.isWindow()){
|
if(CommonDatautils.isWindow()){
|
||||||
@@ -260,9 +261,9 @@ class BigdataPoiBatch extends BaseBatch{
|
|||||||
}else if(type =="丧葬服务"){
|
}else if(type =="丧葬服务"){
|
||||||
ret << "ob101"
|
ret << "ob101"
|
||||||
}else if(type =="学校"){
|
}else if(type =="学校"){
|
||||||
if(d['F'].contains("中学")){
|
if(d['B'].contains("中学")){
|
||||||
ret << "ed138"
|
ret << "ed138"
|
||||||
}else if(d['F'].contains("小学")){
|
}else if(d['B'].contains("小学")){
|
||||||
ret << "ed114"
|
ret << "ed114"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
@@ -417,6 +417,7 @@ class CcbCommonUtils {
|
|||||||
q.put("visible",0)
|
q.put("visible",0)
|
||||||
q.put("cat", cat);
|
q.put("cat", cat);
|
||||||
q.put("city", city);
|
q.put("city", city);
|
||||||
|
// q.put("name","云山尚城")
|
||||||
// q.put("name_cat", 0);
|
// q.put("name_cat", 0);
|
||||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||||
|
|
||||||
|
|||||||
+4
-5
@@ -370,13 +370,12 @@ class DealPriceDataProcess {
|
|||||||
|
|
||||||
|
|
||||||
def deleteThirdPartyData(){
|
def deleteThirdPartyData(){
|
||||||
def citys = CcbCommonUtils.citys
|
def citys = CcbCommonUtils.citys_92
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def month = 202209
|
||||||
def month = 202208
|
|
||||||
def rows = []
|
def rows = []
|
||||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def rets = findPeGteThirdParty(city, 202209)
|
def rets = findPeGteThirdParty(city, 202210)
|
||||||
def total = 0
|
def total = 0
|
||||||
println city +" "+rets.size()
|
println city +" "+rets.size()
|
||||||
for(def m : rets){
|
for(def m : rets){
|
||||||
@@ -388,7 +387,7 @@ class DealPriceDataProcess {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def findPeGteThirdParty(cityName, yearmonth){
|
def findPeGteThirdParty(cityName, yearmonth){
|
||||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30"
|
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25"
|
||||||
def rets = []
|
def rets = []
|
||||||
sqlInstance236.eachRow(sql,{r->
|
sqlInstance236.eachRow(sql,{r->
|
||||||
def map = [:]
|
def map = [:]
|
||||||
|
|||||||
+2
-2
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def citys = CcbCommonUtils.citys;
|
def citys = CcbCommonUtils.citys_252;
|
||||||
def rows = []
|
def rows = []
|
||||||
|
|
||||||
for (def city : citys) {
|
for (def city : citys) {
|
||||||
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
|||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
|
|
||||||
//添加自己库中的人工指导价
|
//添加自己库中的人工指导价
|
||||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-09-22", "2022-09-23")
|
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
|
||||||
if(prices){
|
if(prices){
|
||||||
def distMap = getDistrictMap(city)
|
def distMap = getDistrictMap(city)
|
||||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||||
|
|||||||
+5
-5
@@ -22,10 +22,10 @@ import java.util.concurrent.ConcurrentHashMap
|
|||||||
@BatchOwner(Watcher.GaoJuan)
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
class ThirdPartyPriceBatch extends BaseBatch {
|
class ThirdPartyPriceBatch extends BaseBatch {
|
||||||
|
|
||||||
def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
def startDate = 202207
|
def startDate = 202107
|
||||||
def endDate = 202209
|
def endDate = 202109
|
||||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||||
|
|
||||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||||
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -52,7 +52,7 @@ class YouDaDataParser {
|
|||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new YouDaDataParser();
|
def task = new YouDaDataParser();
|
||||||
// task.newHouseparse();//友达一手房存入mongo
|
// task.newHouseparse();//友达一手房存入mongo
|
||||||
// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo
|
// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo
|
||||||
// task.newhouseToSaveEs();//新房保存到es
|
// task.newhouseToSaveEs();//新房保存到es
|
||||||
// task.esfhouseToSaveEs();//二手房保存到es
|
// task.esfhouseToSaveEs();//二手房保存到es
|
||||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||||
|
|||||||
+12
-4
@@ -49,7 +49,15 @@ public class BeiKeCrawlByChrome {
|
|||||||
|
|
||||||
WebDriver driver = new ChromeDriver()
|
WebDriver driver = new ChromeDriver()
|
||||||
def citys = getAllCityUrls(driver)
|
def citys = getAllCityUrls(driver)
|
||||||
citys = citys.reverse()
|
// citys = citys.reverse()
|
||||||
|
// def index = 0
|
||||||
|
// for(def c : citys){
|
||||||
|
// index++
|
||||||
|
// if(c.city=="合肥"){
|
||||||
|
// break
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// citys = citys[index..citys.size()-1]
|
||||||
def start = 0
|
def start = 0
|
||||||
// for(def i=0;i<citys.size();i++){
|
// for(def i=0;i<citys.size();i++){
|
||||||
// def m = citys[i]
|
// def m = citys[i]
|
||||||
@@ -190,7 +198,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
}
|
}
|
||||||
page++;
|
page++;
|
||||||
}else{
|
}else{
|
||||||
stop = true;
|
stop = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return detailUrls
|
return detailUrls
|
||||||
@@ -390,13 +398,13 @@ public class BeiKeCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles){
|
for(def e : cityEles){
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if(e.text().contains("福州")){
|
// if(e.text().contains("合肥")){
|
||||||
def cityMap = [:]
|
def cityMap = [:]
|
||||||
cityMap.city = e.text()
|
cityMap.city = e.text()
|
||||||
cityMap.url = "https:"+href
|
cityMap.url = "https:"+href
|
||||||
list << cityMap
|
list << cityMap
|
||||||
// cityMap.put(e.text(), "https:"+href)
|
// cityMap.put(e.text(), "https:"+href)
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return list
|
return list
|
||||||
|
|||||||
+1
-1
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles) {
|
for(def e : cityEles) {
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
if (!href.contains("fang.lianjia")){
|
||||||
cityMap.put(e.text(), href)
|
cityMap.put(e.text(), href)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-4
@@ -38,13 +38,13 @@ class PaimaiDataSend {
|
|||||||
// task.jdRunByMongo()
|
// task.jdRunByMongo()
|
||||||
|
|
||||||
// task.aliZcRun()
|
// task.aliZcRun()
|
||||||
// task.aliZcRun2()
|
task.aliZcRun2()
|
||||||
// task.aliFpRun()
|
// task.aliFpRun()
|
||||||
// task.rmfysszcRun()
|
// task.rmfysszcRun()
|
||||||
// task.jdRun()
|
// task.jdRun()
|
||||||
// task.checkUrl()
|
// task.checkUrl()
|
||||||
|
|
||||||
task.paimaiSendToEs()
|
// task.paimaiSendToEs()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -337,7 +337,7 @@ class PaimaiDataSend {
|
|||||||
q.put("province","江苏")
|
q.put("province","江苏")
|
||||||
q.put("sub_type","土地")
|
q.put("sub_type","土地")
|
||||||
// q.put("parser_status", new BasicDBObject('$ne',2))
|
// q.put("parser_status", new BasicDBObject('$ne',2))
|
||||||
// q.put("url","https://sf-item.taobao.com/sf_item/668496148492.htm")
|
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
|
||||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||||
// q.put("parser_status", new BasicDBObject('$exists',false))
|
// q.put("parser_status", new BasicDBObject('$exists',false))
|
||||||
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
||||||
@@ -424,7 +424,7 @@ class PaimaiDataSend {
|
|||||||
q.put("sub_type","土地")
|
q.put("sub_type","土地")
|
||||||
// q.put("parser_status", new BasicDBObject('$ne',1))
|
// q.put("parser_status", new BasicDBObject('$ne',1))
|
||||||
// q.put("region","姑苏")
|
// q.put("region","姑苏")
|
||||||
// q.put("url","https://susong-item.taobao.com/auction/674736290116.htm")
|
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
|
||||||
def dbcursor = ali_zc_coll.find(q).batchSize(1000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
def dbcursor = ali_zc_coll.find(q).batchSize(1000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||||
def rets = []
|
def rets = []
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
|
|||||||
Reference in New Issue
Block a user