This commit is contained in:
2023-02-08 11:19:02 +08:00
parent 039924b31c
commit 3be00b6aaa
6 changed files with 22 additions and 26 deletions
@@ -27,7 +27,7 @@ class BigdataCitypriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def curRunDate = 202212//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
static main(args) {
runBatch(new BigdataCitypriceBatch(), args)
@@ -36,8 +36,8 @@ class BigdataCitypriceBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def startDate = 20221201//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def endDate = 20221231//DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
cityRun(citys, startDate, endDate, cityCodeDataMap)
@@ -30,7 +30,7 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def queryDate = 202212//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
citys = CcbCommonUtils.getRunningCitys(args)
@@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
@@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
//2月8号 临时导出苏州一年的数据
exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出
// exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
@@ -1317,7 +1317,6 @@ class ExportDistrictRongHe extends BaseBatch{
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
def index = 0
citys = ["苏州"]
for(def city : citys){
def cityCode = cityCodeMap.get(city)
@@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202211);
def count2 = findTotalCount(source, 202212);
def count1 = findTotalCount(source, 202212);
def count2 = findTotalCount(source, 202301);
println source +" "+count1 +" "+count2
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.util
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.RandomUtil
import com.fangjia.script.addrepo.InvokeHttpRequest;
import groovy.json.JsonSlurper
@@ -103,24 +104,20 @@ class BaiduMapUtils {
int recount = 0;
while(!stop && recount<3) {
try{
def ret = ABuYunUtil.doGet(url, null, null, "utf-8");
if (ret) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if(jsonObj.getInt("status") == 121){
println "key over"
changeAK()
recount++;
HttpRequest request = new HttpRequest();
try{
String result = request.get(url,null,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
if(json.getInt("status") >= 300){
changeAK();
json = searchByAddress(address);
}else{
def result = jsonObj.result;
if(result){
stop = true
return result;
}else{
stop = true
}
return json.result
}
}catch (Exception e){
e.printStackTrace();
}
return null;
}catch (Exception e){
recount++;
println e.getMessage()
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "90010d53a16ce9df544acab297645106"
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390"
/**
* 经纬度转地址<br>