+3
-3
@@ -27,7 +27,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def curRunDate = 202212//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
|
||||
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCitypriceBatch(), args)
|
||||
@@ -36,8 +36,8 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def startDate = 20221201//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
|
||||
def endDate = 20221231//DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
|
||||
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
|
||||
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
cityRun(citys, startDate, endDate, cityCodeDataMap)
|
||||
|
||||
+3
-4
@@ -30,7 +30,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def queryDate = 202212//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
|
||||
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
@@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def new_dists = []//["350604","350605","410307","410308","450181"]
|
||||
//按天导(每天一个文件夹)
|
||||
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
|
||||
|
||||
if(curDate.toString().endsWith("10")){
|
||||
@@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
//2月8号 临时导出苏州一年的数据
|
||||
exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出
|
||||
// exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出
|
||||
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
@@ -1317,7 +1317,6 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
|
||||
def index = 0
|
||||
citys = ["苏州"]
|
||||
for(def city : citys){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
|
||||
def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580]
|
||||
static main(args) {
|
||||
runBatch(new DistrictPriceExportBatch(), args)
|
||||
}
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202211);
|
||||
def count2 = findTotalCount(source, 202212);
|
||||
def count1 = findTotalCount(source, 202212);
|
||||
def count2 = findTotalCount(source, 202301);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+12
-15
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.RandomUtil
|
||||
import com.fangjia.script.addrepo.InvokeHttpRequest;
|
||||
import groovy.json.JsonSlurper
|
||||
@@ -103,24 +104,20 @@ class BaiduMapUtils {
|
||||
int recount = 0;
|
||||
while(!stop && recount<3) {
|
||||
try{
|
||||
def ret = ABuYunUtil.doGet(url, null, null, "utf-8");
|
||||
if (ret) {
|
||||
JSONObject jsonObj = JSONObject.fromObject(ret);
|
||||
|
||||
if(jsonObj.getInt("status") == 121){
|
||||
println "key over"
|
||||
changeAK()
|
||||
recount++;
|
||||
HttpRequest request = new HttpRequest();
|
||||
try{
|
||||
String result = request.get(url,null,"UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(result);
|
||||
if(json.getInt("status") >= 300){
|
||||
changeAK();
|
||||
json = searchByAddress(address);
|
||||
}else{
|
||||
def result = jsonObj.result;
|
||||
if(result){
|
||||
stop = true
|
||||
return result;
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
return json.result
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}catch (Exception e){
|
||||
recount++;
|
||||
println e.getMessage()
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "90010d53a16ce9df544acab297645106"
|
||||
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390"
|
||||
|
||||
/**
|
||||
* 经纬度转地址<br>
|
||||
|
||||
Reference in New Issue
Block a user