Merge branch 'script_gj' into 'master'
福州网站及使用期限修改 See merge request !33
This commit is contained in:
+3
@@ -165,6 +165,9 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
rets << "非钢混"
|
||||
}
|
||||
}
|
||||
if(rets.size()==0){
|
||||
rets << "钢混"
|
||||
}
|
||||
return rets.join(",")
|
||||
}
|
||||
return "钢混";
|
||||
|
||||
+5
-5
@@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils
|
||||
*/
|
||||
class ThirdPriceExportBatch extends BaseBatch{
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
@@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
||||
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
||||
}
|
||||
if(!retMap.fangtan_esf_price && d.fangtan_esf_price){
|
||||
retMap.fangtan_esf_price = d.fangtan_esf_price
|
||||
if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取
|
||||
retMap.fangtan_esf_price = d.jiwu_esf_price
|
||||
}
|
||||
|
||||
if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){
|
||||
retMap.fangdoudou_esf_price = d.fangdoudou_esf_price
|
||||
if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多
|
||||
retMap.fangdoudou_esf_price = d.get("365house_esf_price")
|
||||
}
|
||||
|
||||
if(!retMap.maitian_esf_price && d.maitian_esf_price){
|
||||
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-03-01"
|
||||
static def statDate = "2021-04-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+5
-1
@@ -9,7 +9,11 @@ class FangjiaClientUtils {
|
||||
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
||||
JSONObject jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
if(address){
|
||||
url = url + "&address="+address
|
||||
}
|
||||
// def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
|
||||
+5
@@ -25,6 +25,7 @@ class CrawalDBUtil {
|
||||
static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process")
|
||||
static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics")
|
||||
static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process")
|
||||
static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||
|
||||
public static getSeedsCol() {
|
||||
return seedsColl;
|
||||
@@ -67,6 +68,10 @@ class CrawalDBUtil {
|
||||
return map_district_col;
|
||||
}
|
||||
|
||||
public static getDistrictDetailColl(){
|
||||
return districtDetailColl;
|
||||
}
|
||||
|
||||
static def updateSeedsCount(def id, def count) {
|
||||
|
||||
DBObject uquery = new BasicDBObject()
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
static {
|
||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf"
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf"
|
||||
]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||
|
||||
|
||||
@@ -309,6 +309,31 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject addressStandard(String city, String address) {
|
||||
if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content)
|
||||
if (result.getInt("code") == 200) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") == 500) {
|
||||
println("${city} ${address} ${result}")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${address} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
@@ -1214,4 +1239,6 @@ public class OpenApiClient {
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+26
@@ -223,6 +223,32 @@ class GaoDeMapApiData {
|
||||
doSearchByGaoDe(type, lng, lat, radius, page)
|
||||
}
|
||||
}
|
||||
|
||||
def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("key", GAO_DE_AK)
|
||||
params.put("types", type)
|
||||
params.put("location", Double.toString(lng)+ ","+ Double.toString(lat))
|
||||
params.put("radius",radius)
|
||||
params.put("offset",pageSize)
|
||||
params.put("page",page)
|
||||
String content
|
||||
try {
|
||||
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||
}
|
||||
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||
|
||||
if(jsonObj.get("infocode").equals("10000")){
|
||||
JSONArray pois = jsonObj.get("pois")
|
||||
return pois
|
||||
} else {
|
||||
println jsonObj.get("infocode")+"exit........"
|
||||
doSearchByGaoDe(type, lng, lat, radius, page, pageSize)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private synchronized static fzData(JSONArray pois) {
|
||||
pois.each { poi ->
|
||||
|
||||
Reference in New Issue
Block a user