Merge branch 'script_gj' into 'master'
Script gj See merge request !427
This commit is contained in:
@@ -242,5 +242,13 @@
|
|||||||
<version>1.0.0</version>
|
<version>1.0.0</version>
|
||||||
<classifier>jdk18</classifier>
|
<classifier>jdk18</classifier>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<!-- <dependency>-->
|
||||||
|
<!-- <groupId>org.springframework.boot</groupId>-->
|
||||||
|
<!-- <artifactId>spring-boot-devtools</artifactId>-->
|
||||||
|
<!-- <!– optional 表示依赖是否向下传递 true表示不向下传递 默认值是false向下传递 –>-->
|
||||||
|
<!-- <optional>true</optional>-->
|
||||||
|
<!-- </dependency>-->
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
@@ -38,7 +38,9 @@
|
|||||||
<a href="${wwwUrl}/resource/industry" target="_blank">行业动态</a>
|
<a href="${wwwUrl}/resource/industry" target="_blank">行业动态</a>
|
||||||
<a href="${wwwUrl}/resource/analysis" target="_blank">数据分析</a>
|
<a href="${wwwUrl}/resource/analysis" target="_blank">数据分析</a>
|
||||||
<a href="${wwwUrl}/resource/case" target="_blank">成功案例</a>
|
<a href="${wwwUrl}/resource/case" target="_blank">成功案例</a>
|
||||||
<a href="${wwwUrl}/resource/company" target="_blank">公司动态</a>
|
<a href="https://sh.fangjia.com/" target="_blank">各路客</a>
|
||||||
|
<a href="http://user.fangjia.com/" target="_blank">房价通</a>
|
||||||
|
<a href="https://www.didatai.com/" target="_blank">地数</a>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
+2
-2
@@ -68,14 +68,14 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def index = 0
|
def index = 0
|
||||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||||
statCity(citys)
|
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
index++
|
index++
|
||||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019);
|
||||||
|
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221019);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||||
//测试通过匹配接口配房源效果
|
//测试通过匹配接口配房源效果
|
||||||
|
|||||||
+5
-5
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
|||||||
@BatchOwner(Watcher.GaoJuan)
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
class ThirdPartyPriceBatch extends BaseBatch {
|
class ThirdPartyPriceBatch extends BaseBatch {
|
||||||
|
|
||||||
def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
def startDate = 202107
|
def startDate = 202207
|
||||||
def endDate = 202109
|
def endDate = 202209
|
||||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||||
|
|
||||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||||
@@ -502,8 +502,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
|||||||
def findThirdPartyPrice(def city, def statMap){
|
def findThirdPartyPrice(def city, def statMap){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
|
q.put("s_date", endDate)
|
||||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
// q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||||
def map = [:]
|
def map = [:]
|
||||||
for(def db : dbcursor){
|
for(def db : dbcursor){
|
||||||
|
|||||||
+37
-12
@@ -41,10 +41,18 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
|
|
||||||
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
|
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
|
||||||
|
|
||||||
def ret = createDatas("福州")
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
citys.remove("福州");
|
||||||
saveToDB(ret)
|
citys.add(0, "福州")
|
||||||
|
|
||||||
|
def fz_datas = []
|
||||||
|
for(def city : citys){
|
||||||
|
def ret = createDatas("福州")
|
||||||
|
saveToDB(ret)
|
||||||
|
if(city == "福州"){
|
||||||
|
fz_datas = ret
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||||
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||||
@@ -52,7 +60,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||||
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||||
}
|
}
|
||||||
def sb = appendResult(headers, ret)
|
def sb = appendResult(headers, fz_datas)
|
||||||
DataHandleUtils.outPutFile(fileName, sb, charset);
|
DataHandleUtils.outPutFile(fileName, sb, charset);
|
||||||
FzFtpUtils.ftpUpload(fileName)
|
FzFtpUtils.ftpUpload(fileName)
|
||||||
//生成加密数据文件并上传
|
//生成加密数据文件并上传
|
||||||
@@ -102,7 +110,12 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
// def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||||
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
// def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01");
|
||||||
// districts.addAll(removeDists)
|
// districts.addAll(removeDists)
|
||||||
def districts = getFzDistricts(city)
|
def districts = []
|
||||||
|
if(city == "福州"){
|
||||||
|
districts = getFzDistricts(city)
|
||||||
|
}else{
|
||||||
|
districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||||
|
}
|
||||||
|
|
||||||
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
||||||
def dataStr = DateUtils.add(endDateStr, -30)
|
def dataStr = DateUtils.add(endDateStr, -30)
|
||||||
@@ -127,14 +140,26 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
curPrice = dbo.get("avg_price")
|
curPrice = dbo.get("avg_price")
|
||||||
if(lastMonthPrice && curPrice && dbo.value_type!=1){
|
if(lastMonthPrice && curPrice && dbo.value_type!=1){
|
||||||
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
||||||
if(Math.abs(markup)>3){
|
if(city == "福州"){
|
||||||
def randMarkup = nextDouble(0, 3)
|
if(Math.abs(markup)>3){
|
||||||
if(markup < 0){
|
def randMarkup = nextDouble(0, 3)
|
||||||
markup = randMarkup*(-1)
|
if(markup < 0){
|
||||||
}else{
|
markup = randMarkup*(-1)
|
||||||
markup = randMarkup
|
}else{
|
||||||
|
markup = randMarkup
|
||||||
|
}
|
||||||
|
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(Math.abs(markup)>10){
|
||||||
|
def randMarkup = nextDouble(0, 10)
|
||||||
|
if(markup < 0){
|
||||||
|
markup = randMarkup*(-10)
|
||||||
|
}else{
|
||||||
|
markup = randMarkup
|
||||||
|
}
|
||||||
|
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||||
}
|
}
|
||||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
|
|||||||
+11
-9
@@ -1,6 +1,7 @@
|
|||||||
package com.fangjia.script.temp.shenjian.poi
|
package com.fangjia.script.temp.shenjian.poi
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject
|
import com.alibaba.fastjson.JSONObject
|
||||||
|
import com.fangjia.script.temp.shenjian.util.BaiDuUtils
|
||||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
@@ -15,25 +16,26 @@ import java.util.concurrent.Executors
|
|||||||
*/
|
*/
|
||||||
class FillLngLat {
|
class FillLngLat {
|
||||||
|
|
||||||
static ExecutorService executorService = Executors.newFixedThreadPool(4)
|
// static ExecutorService executorService = Executors.newFixedThreadPool(4)
|
||||||
static void main(String[] args) {
|
static void main(String[] args) {
|
||||||
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
|
ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { dataT ->
|
||||||
final def data = dataT
|
final def data = dataT
|
||||||
executorService.submit(new Runnable() {
|
// executorService.submit(new Runnable() {
|
||||||
@Override
|
// @Override
|
||||||
void run() {
|
// void run() {
|
||||||
println(data)
|
println(data)
|
||||||
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
|
def lngLat = BaiDuUtils.parserByBaidu(data.city, data.region + data.name + data.address)
|
||||||
|
Thread.sleep(1000)
|
||||||
if (lngLat != null && lngLat.size() == 2 ) {
|
if (lngLat != null && lngLat.size() == 2 ) {
|
||||||
JSONObject jsonObject = new JSONObject()
|
JSONObject jsonObject = new JSONObject()
|
||||||
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
|
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
|
||||||
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
|
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
|
||||||
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
|
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
|
||||||
}
|
}
|
||||||
}
|
// }
|
||||||
})
|
// })
|
||||||
|
|
||||||
})
|
})
|
||||||
executorService.shutdown()
|
//executorService.shutdown()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+48
@@ -0,0 +1,48 @@
|
|||||||
|
package com.fangjia.script.temp.shenjian.util
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONObject
|
||||||
|
import com.fangjia.core.util.GeoUtils
|
||||||
|
import com.fangjia.core.util.HttpRequest
|
||||||
|
import com.fangjia.script.util.DataUtil
|
||||||
|
import org.apache.commons.lang.StringUtils
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 百度工具类
|
||||||
|
*
|
||||||
|
* @author shenjian
|
||||||
|
* @since 2022/10/28
|
||||||
|
*/
|
||||||
|
class BaiDuUtils {
|
||||||
|
|
||||||
|
def static final String KEY = "6w9SEHS3hnAFjmY2oEhqIDdgvLPKWXu5"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 百度地址逆解析,返回高德坐标!!!!!!!!!!!!!!!
|
||||||
|
* @param city
|
||||||
|
* @param address
|
||||||
|
*/
|
||||||
|
def static parserByBaidu(def city, def address) {
|
||||||
|
Map<String, String> params = new HashMap<String, String>()
|
||||||
|
params.put("city", city)
|
||||||
|
if (StringUtils.isBlank(address)) {
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
params.put("address", address.replaceAll("null", ""))
|
||||||
|
params.put("output", "json")
|
||||||
|
params.put("ak", KEY)
|
||||||
|
params.put("callback", "showLocation")
|
||||||
|
|
||||||
|
String content
|
||||||
|
content = new HttpRequest().get("https://api.map.baidu.com/geocoding/v3/", params, "utf-8")
|
||||||
|
//content = "showLocation&&showLocation({\"status\":0,\"result\":{\"location\":{\"lng\":131.04009309945693,\"lat\":45.78426770180344},\"precise\":1,\"confidence\":70,\"comprehension\":100,\"level\":\"地产小区\"}})"
|
||||||
|
content = content.replaceAll("showLocation&&showLocation\\(", "").replaceAll("\\)", "")
|
||||||
|
JSONObject json = JSONObject.parseObject(content)
|
||||||
|
if (json.status == 0 && json.result && json.result.location) {
|
||||||
|
def bd_lng = json.result.location.lng
|
||||||
|
def bd_lat = json.result.location.lat
|
||||||
|
Map<String, Double> locationMap = GeoUtils.bd09ToGcj02(bd_lat, bd_lng)
|
||||||
|
return ["lng": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lng']), "lat": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lat'])]
|
||||||
|
}
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user