Merge branch 'script_gj' into 'master'

Script gj

See merge request !427
This commit is contained in:
2022-11-05 11:34:54 +08:00
7 changed files with 114 additions and 29 deletions
@@ -68,14 +68,14 @@ class BigdataForsaleBatch extends BaseBatch{
public void process(Object args) {
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
statCity(citys)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221019);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202107
def endDate = 202109
def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202207
def endDate = 202209
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -502,8 +502,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
q.put("s_date", endDate)
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
@@ -41,10 +41,18 @@ class DistrictPriceExportBatch extends BaseBatch{
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
def ret = createDatas("福州")
saveToDB(ret)
def citys = CommonDatautils.findAllCity()
citys.remove("福州");
citys.add(0, "福州")
def fz_datas = []
for(def city : citys){
def ret = createDatas("福州")
saveToDB(ret)
if(city == "福州"){
fz_datas = ret
}
}
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
@@ -52,7 +60,7 @@ class DistrictPriceExportBatch extends BaseBatch{
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
}
def sb = appendResult(headers, ret)
def sb = appendResult(headers, fz_datas)
DataHandleUtils.outPutFile(fileName, sb, charset);
FzFtpUtils.ftpUpload(fileName)
//生成加密数据文件并上传
@@ -102,7 +110,12 @@ class DistrictPriceExportBatch extends BaseBatch{
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
// districts.addAll(removeDists)
def districts = getFzDistricts(city)
def districts = []
if(city == "福州"){
districts = getFzDistricts(city)
}else{
districts = CommonDatautils.findAllDistricts(city, "district", null);
}
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
@@ -127,14 +140,26 @@ class DistrictPriceExportBatch extends BaseBatch{
curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice && dbo.value_type!=1){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
if(markup < 0){
markup = randMarkup*(-1)
}else{
markup = randMarkup
if(city == "福州"){
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
if(markup < 0){
markup = randMarkup*(-1)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}else{
if(Math.abs(markup)>10){
def randMarkup = nextDouble(0, 10)
if(markup < 0){
markup = randMarkup*(-10)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
}else{
@@ -1,6 +1,7 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.BaiDuUtils
import com.fangjia.script.temp.shenjian.util.EsUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ElasticUtils
@@ -15,25 +16,26 @@ import java.util.concurrent.Executors
*/
class FillLngLat {
static ExecutorService executorService = Executors.newFixedThreadPool(4)
// static ExecutorService executorService = Executors.newFixedThreadPool(4)
static void main(String[] args) {
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { dataT ->
final def data = dataT
executorService.submit(new Runnable() {
@Override
void run() {
// executorService.submit(new Runnable() {
// @Override
// void run() {
println(data)
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
def lngLat = BaiDuUtils.parserByBaidu(data.city, data.region + data.name + data.address)
Thread.sleep(1000)
if (lngLat != null && lngLat.size() == 2 ) {
JSONObject jsonObject = new JSONObject()
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
}
}
})
// }
// })
})
executorService.shutdown()
//executorService.shutdown()
}
}
@@ -0,0 +1,48 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.DataUtil
import org.apache.commons.lang.StringUtils
/**
* 百度工具类
*
* @author shenjian
* @since 2022/10/28
*/
class BaiDuUtils {
def static final String KEY = "6w9SEHS3hnAFjmY2oEhqIDdgvLPKWXu5"
/**
* 百度地址逆解析,返回高德坐标!!!!!!!!!!!!!!!
* @param city
* @param address
*/
def static parserByBaidu(def city, def address) {
Map<String, String> params = new HashMap<String, String>()
params.put("city", city)
if (StringUtils.isBlank(address)) {
return []
}
params.put("address", address.replaceAll("null", ""))
params.put("output", "json")
params.put("ak", KEY)
params.put("callback", "showLocation")
String content
content = new HttpRequest().get("https://api.map.baidu.com/geocoding/v3/", params, "utf-8")
//content = "showLocation&&showLocation({\"status\":0,\"result\":{\"location\":{\"lng\":131.04009309945693,\"lat\":45.78426770180344},\"precise\":1,\"confidence\":70,\"comprehension\":100,\"level\":\"地产小区\"}})"
content = content.replaceAll("showLocation&&showLocation\\(", "").replaceAll("\\)", "")
JSONObject json = JSONObject.parseObject(content)
if (json.status == 0 && json.result && json.result.location) {
def bd_lng = json.result.location.lng
def bd_lat = json.result.location.lat
Map<String, Double> locationMap = GeoUtils.bd09ToGcj02(bd_lat, bd_lng)
return ["lng": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lng']), "lat": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lat'])]
}
return []
}
}