Merge branch 'script_gj' into 'master'
Script gj See merge request !427
This commit is contained in:
+2
-2
@@ -68,14 +68,14 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
def index = 0
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
statCity(citys)
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221019);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
|
||||
+5
-5
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202107
|
||||
def endDate = 202109
|
||||
def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202207
|
||||
def endDate = 202209
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
@@ -502,8 +502,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
q.put("s_date", endDate)
|
||||
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
|
||||
+37
-12
@@ -41,10 +41,18 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
|
||||
|
||||
def ret = createDatas("福州")
|
||||
|
||||
saveToDB(ret)
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys.remove("福州");
|
||||
citys.add(0, "福州")
|
||||
|
||||
def fz_datas = []
|
||||
for(def city : citys){
|
||||
def ret = createDatas("福州")
|
||||
saveToDB(ret)
|
||||
if(city == "福州"){
|
||||
fz_datas = ret
|
||||
}
|
||||
}
|
||||
|
||||
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
@@ -52,7 +60,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
}
|
||||
def sb = appendResult(headers, ret)
|
||||
def sb = appendResult(headers, fz_datas)
|
||||
DataHandleUtils.outPutFile(fileName, sb, charset);
|
||||
FzFtpUtils.ftpUpload(fileName)
|
||||
//生成加密数据文件并上传
|
||||
@@ -102,7 +110,12 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||
// districts.addAll(removeDists)
|
||||
def districts = getFzDistricts(city)
|
||||
def districts = []
|
||||
if(city == "福州"){
|
||||
districts = getFzDistricts(city)
|
||||
}else{
|
||||
districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
}
|
||||
|
||||
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
||||
def dataStr = DateUtils.add(endDateStr, -30)
|
||||
@@ -127,14 +140,26 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
curPrice = dbo.get("avg_price")
|
||||
if(lastMonthPrice && curPrice && dbo.value_type!=1){
|
||||
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
||||
if(Math.abs(markup)>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(markup < 0){
|
||||
markup = randMarkup*(-1)
|
||||
}else{
|
||||
markup = randMarkup
|
||||
if(city == "福州"){
|
||||
if(Math.abs(markup)>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(markup < 0){
|
||||
markup = randMarkup*(-1)
|
||||
}else{
|
||||
markup = randMarkup
|
||||
}
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
}else{
|
||||
if(Math.abs(markup)>10){
|
||||
def randMarkup = nextDouble(0, 10)
|
||||
if(markup < 0){
|
||||
markup = randMarkup*(-10)
|
||||
}else{
|
||||
markup = randMarkup
|
||||
}
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
|
||||
+11
-9
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.BaiDuUtils
|
||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
@@ -15,25 +16,26 @@ import java.util.concurrent.Executors
|
||||
*/
|
||||
class FillLngLat {
|
||||
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(4)
|
||||
// static ExecutorService executorService = Executors.newFixedThreadPool(4)
|
||||
static void main(String[] args) {
|
||||
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
|
||||
ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { dataT ->
|
||||
final def data = dataT
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
// executorService.submit(new Runnable() {
|
||||
// @Override
|
||||
// void run() {
|
||||
println(data)
|
||||
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
|
||||
def lngLat = BaiDuUtils.parserByBaidu(data.city, data.region + data.name + data.address)
|
||||
Thread.sleep(1000)
|
||||
if (lngLat != null && lngLat.size() == 2 ) {
|
||||
JSONObject jsonObject = new JSONObject()
|
||||
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
|
||||
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
|
||||
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
|
||||
}
|
||||
}
|
||||
})
|
||||
// }
|
||||
// })
|
||||
|
||||
})
|
||||
executorService.shutdown()
|
||||
//executorService.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package com.fangjia.script.temp.shenjian.util
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
/**
|
||||
* 百度工具类
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2022/10/28
|
||||
*/
|
||||
class BaiDuUtils {
|
||||
|
||||
def static final String KEY = "6w9SEHS3hnAFjmY2oEhqIDdgvLPKWXu5"
|
||||
|
||||
/**
|
||||
* 百度地址逆解析,返回高德坐标!!!!!!!!!!!!!!!
|
||||
* @param city
|
||||
* @param address
|
||||
*/
|
||||
def static parserByBaidu(def city, def address) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("city", city)
|
||||
if (StringUtils.isBlank(address)) {
|
||||
return []
|
||||
}
|
||||
params.put("address", address.replaceAll("null", ""))
|
||||
params.put("output", "json")
|
||||
params.put("ak", KEY)
|
||||
params.put("callback", "showLocation")
|
||||
|
||||
String content
|
||||
content = new HttpRequest().get("https://api.map.baidu.com/geocoding/v3/", params, "utf-8")
|
||||
//content = "showLocation&&showLocation({\"status\":0,\"result\":{\"location\":{\"lng\":131.04009309945693,\"lat\":45.78426770180344},\"precise\":1,\"confidence\":70,\"comprehension\":100,\"level\":\"地产小区\"}})"
|
||||
content = content.replaceAll("showLocation&&showLocation\\(", "").replaceAll("\\)", "")
|
||||
JSONObject json = JSONObject.parseObject(content)
|
||||
if (json.status == 0 && json.result && json.result.location) {
|
||||
def bd_lng = json.result.location.lng
|
||||
def bd_lat = json.result.location.lat
|
||||
Map<String, Double> locationMap = GeoUtils.bd09ToGcj02(bd_lat, bd_lng)
|
||||
return ["lng": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lng']), "lat": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lat'])]
|
||||
}
|
||||
return []
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user