Merge branch 'script_gj' into 'master'

福州网站及使用期限修改

See merge request !33
This commit is contained in:
2021-05-24 09:11:20 +08:00
9 changed files with 75 additions and 10 deletions
@@ -165,6 +165,9 @@ class DistrictInfoExportBatch extends BaseBatch{
rets << "非钢混"
}
}
if(rets.size()==0){
rets << "钢混"
}
return rets.join(",")
}
return "钢混";
@@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils
*/
class ThirdPriceExportBatch extends BaseBatch{
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
retMap.cityhouse_esf_price = d.cityhouse_esf_price
}
if(!retMap.fangtan_esf_price && d.fangtan_esf_price){
retMap.fangtan_esf_price = d.fangtan_esf_price
if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取
retMap.fangtan_esf_price = d.jiwu_esf_price
}
if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){
retMap.fangdoudou_esf_price = d.fangdoudou_esf_price
if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多
retMap.fangdoudou_esf_price = d.get("365house_esf_price")
}
if(!retMap.maitian_esf_price && d.maitian_esf_price){
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-03-01"
static def statDate = "2021-04-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -9,7 +9,11 @@ class FangjiaClientUtils {
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
JSONObject jsonObj = null
try {
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
if(address){
url = url + "&address="+address
}
// def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
def content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
@@ -25,6 +25,7 @@ class CrawalDBUtil {
static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process")
static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics")
static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process")
static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
public static getSeedsCol() {
return seedsColl;
@@ -67,6 +68,10 @@ class CrawalDBUtil {
return map_district_col;
}
public static getDistrictDetailColl(){
return districtDetailColl;
}
static def updateSeedsCount(def id, def count) {
DBObject uquery = new BasicDBObject()
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
def static final SHORT_DATE = Utils.getLastMonth()
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static {
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
@@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf"
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf"
]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
@@ -309,6 +309,31 @@ public class OpenApiClient {
return null
}
public static JSONObject addressStandard(String city, String address) {
if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) {
throw new IllegalArgumentException();
}
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content)
if (result.getInt("code") == 200) {
return result.getJSONObject("result");
} else if (result.getInt("code") == 500) {
println("${city} ${address} ${result}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) {
try {
Map<String, String> params = new HashMap<String, String>();
@@ -1214,4 +1239,6 @@ public class OpenApiClient {
return null;
}
}
@@ -223,6 +223,32 @@ class GaoDeMapApiData {
doSearchByGaoDe(type, lng, lat, radius, page)
}
}
def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("types", type)
params.put("location", Double.toString(lng)+ ","+ Double.toString(lat))
params.put("radius",radius)
params.put("offset",pageSize)
params.put("page",page)
String content
try {
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
}
JSONObject jsonObj = JSONObject.fromObject(content)
if(jsonObj.get("infocode").equals("10000")){
JSONArray pois = jsonObj.get("pois")
return pois
} else {
println jsonObj.get("infocode")+"exit........"
doSearchByGaoDe(type, lng, lat, radius, page, pageSize)
}
return null
}
private synchronized static fzData(JSONArray pois) {
pois.each { poi ->