Merge branch 'script_gj' into 'master'
福州网站及使用期限修改 See merge request !33
This commit is contained in:
+3
@@ -165,6 +165,9 @@ class DistrictInfoExportBatch extends BaseBatch{
|
|||||||
rets << "非钢混"
|
rets << "非钢混"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if(rets.size()==0){
|
||||||
|
rets << "钢混"
|
||||||
|
}
|
||||||
return rets.join(",")
|
return rets.join(",")
|
||||||
}
|
}
|
||||||
return "钢混";
|
return "钢混";
|
||||||
|
|||||||
+5
-5
@@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils
|
|||||||
*/
|
*/
|
||||||
class ThirdPriceExportBatch extends BaseBatch{
|
class ThirdPriceExportBatch extends BaseBatch{
|
||||||
|
|
||||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
|
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
|
|
||||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||||
@@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{
|
|||||||
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
||||||
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
||||||
}
|
}
|
||||||
if(!retMap.fangtan_esf_price && d.fangtan_esf_price){
|
if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取
|
||||||
retMap.fangtan_esf_price = d.fangtan_esf_price
|
retMap.fangtan_esf_price = d.jiwu_esf_price
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){
|
if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多
|
||||||
retMap.fangdoudou_esf_price = d.fangdoudou_esf_price
|
retMap.fangdoudou_esf_price = d.get("365house_esf_price")
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!retMap.maitian_esf_price && d.maitian_esf_price){
|
if(!retMap.maitian_esf_price && d.maitian_esf_price){
|
||||||
|
|||||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
|||||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||||
|
|
||||||
|
|
||||||
static def statDate = "2021-03-01"
|
static def statDate = "2021-04-01"
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new YouDaDataParser();
|
def task = new YouDaDataParser();
|
||||||
// task.newHouseparse();//友达一手房存入mongo
|
// task.newHouseparse();//友达一手房存入mongo
|
||||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
|||||||
|
|
||||||
|
|
||||||
def esfParser(){
|
def esfParser(){
|
||||||
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
|
def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"]
|
||||||
def datas = []
|
def datas = []
|
||||||
for(def fileName : files){
|
for(def fileName : files){
|
||||||
def rows = readDBF(fileName);
|
def rows = readDBF(fileName);
|
||||||
|
|||||||
+5
-1
@@ -9,7 +9,11 @@ class FangjiaClientUtils {
|
|||||||
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
||||||
JSONObject jsonObj = null
|
JSONObject jsonObj = null
|
||||||
try {
|
try {
|
||||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||||
|
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||||
|
if(address){
|
||||||
|
url = url + "&address="+address
|
||||||
|
}
|
||||||
// def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
// def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
||||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||||
jsonObj = JSONObject.fromObject(content)
|
jsonObj = JSONObject.fromObject(content)
|
||||||
|
|||||||
+5
@@ -25,6 +25,7 @@ class CrawalDBUtil {
|
|||||||
static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process")
|
static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process")
|
||||||
static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics")
|
static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics")
|
||||||
static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process")
|
static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process")
|
||||||
|
static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||||
|
|
||||||
public static getSeedsCol() {
|
public static getSeedsCol() {
|
||||||
return seedsColl;
|
return seedsColl;
|
||||||
@@ -67,6 +68,10 @@ class CrawalDBUtil {
|
|||||||
return map_district_col;
|
return map_district_col;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static getDistrictDetailColl(){
|
||||||
|
return districtDetailColl;
|
||||||
|
}
|
||||||
|
|
||||||
static def updateSeedsCount(def id, def count) {
|
static def updateSeedsCount(def id, def count) {
|
||||||
|
|
||||||
DBObject uquery = new BasicDBObject()
|
DBObject uquery = new BasicDBObject()
|
||||||
|
|||||||
+1
-1
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def static final SHORT_DATE = Utils.getLastMonth()
|
def static final SHORT_DATE = Utils.getLastMonth()
|
||||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
|
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
|
||||||
|
|
||||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
static {
|
static {
|
||||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||||
|
|||||||
+1
-1
@@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
|||||||
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
||||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf"
|
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf"
|
||||||
]
|
]
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||||
|
|
||||||
|
|||||||
@@ -309,6 +309,31 @@ public class OpenApiClient {
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static JSONObject addressStandard(String city, String address) {
|
||||||
|
if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) {
|
||||||
|
throw new IllegalArgumentException();
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
Map<String, String> params = new HashMap<String, String>();
|
||||||
|
params.put("city", city);
|
||||||
|
params.put("address", address);
|
||||||
|
params.put("token", TOKEN);
|
||||||
|
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
|
||||||
|
JSONObject result = JSONObject.fromObject(content)
|
||||||
|
if (result.getInt("code") == 200) {
|
||||||
|
return result.getJSONObject("result");
|
||||||
|
} else if (result.getInt("code") == 500) {
|
||||||
|
println("${city} ${address} ${result}")
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
println("${city} ${address} ${e.getMessage()}")
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) {
|
public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) {
|
||||||
try {
|
try {
|
||||||
Map<String, String> params = new HashMap<String, String>();
|
Map<String, String> params = new HashMap<String, String>();
|
||||||
@@ -1214,4 +1239,6 @@ public class OpenApiClient {
|
|||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+26
@@ -224,6 +224,32 @@ class GaoDeMapApiData {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) {
|
||||||
|
Map<String, String> params = new HashMap<String, String>()
|
||||||
|
params.put("key", GAO_DE_AK)
|
||||||
|
params.put("types", type)
|
||||||
|
params.put("location", Double.toString(lng)+ ","+ Double.toString(lat))
|
||||||
|
params.put("radius",radius)
|
||||||
|
params.put("offset",pageSize)
|
||||||
|
params.put("page",page)
|
||||||
|
String content
|
||||||
|
try {
|
||||||
|
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||||
|
} catch (Exception e) {
|
||||||
|
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||||
|
}
|
||||||
|
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||||
|
|
||||||
|
if(jsonObj.get("infocode").equals("10000")){
|
||||||
|
JSONArray pois = jsonObj.get("pois")
|
||||||
|
return pois
|
||||||
|
} else {
|
||||||
|
println jsonObj.get("infocode")+"exit........"
|
||||||
|
doSearchByGaoDe(type, lng, lat, radius, page, pageSize)
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
private synchronized static fzData(JSONArray pois) {
|
private synchronized static fzData(JSONArray pois) {
|
||||||
pois.each { poi ->
|
pois.each { poi ->
|
||||||
try {
|
try {
|
||||||
|
|||||||
Reference in New Issue
Block a user