Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -25,6 +25,7 @@ public class RegionData implements PriorityOrdered {
|
||||
private final static Logger LOGGER = LoggerFactory.getLogger(RegionData.class);
|
||||
private final static long REGION_RELOAD_INTERVAL = 1000;
|
||||
private static DBCollection REGION_CONN = null;
|
||||
private static DBCollection TOWN_CONN = null;
|
||||
private static Set<String> ALL_CITY = new HashSet<String>();
|
||||
private static Map<String,List<String>> cityAndAliasMap = new HashMap<String,List<String>>();
|
||||
|
||||
@@ -32,12 +33,15 @@ public class RegionData implements PriorityOrdered {
|
||||
* 城市对应级别Map
|
||||
*/
|
||||
private static Map<String, Integer> cityGradeMap = new HashMap<String, Integer>();
|
||||
|
||||
protected final static Map<String, Map<String, Map<String, Map<String, List<String>>>>> townDataMap = new HashMap<String, Map<String, Map<String, Map<String, List<String>>>>>();
|
||||
|
||||
private static void init() {
|
||||
|
||||
try {
|
||||
REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
|
||||
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
|
||||
TOWN_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
|
||||
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
|
||||
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1).append("alias", 1));
|
||||
BasicDBObject dbo;
|
||||
while(cursor.hasNext()){
|
||||
@@ -58,6 +62,7 @@ public class RegionData implements PriorityOrdered {
|
||||
|
||||
start(REGION_RELOAD_INTERVAL);
|
||||
reload();
|
||||
// reloadTown();
|
||||
}
|
||||
|
||||
private static void start(long time) {
|
||||
@@ -114,6 +119,66 @@ public class RegionData implements PriorityOrdered {
|
||||
|
||||
LOGGER.info("RegionData reload() size [" + regionDataMap.size() + "]");
|
||||
}
|
||||
public static void reloadTown(){
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put("year", 2022);
|
||||
|
||||
DBObject fields = new BasicDBObject();
|
||||
fields.put("fangjia_city", 1);
|
||||
fields.put("fangjia_area", 1);
|
||||
fields.put("street_name", 1);
|
||||
fields.put("alias", 1);
|
||||
|
||||
|
||||
DBCursor cursor = TOWN_CONN.find(query, fields);
|
||||
while (cursor.hasNext()) {
|
||||
DBObject dbo = cursor.next();
|
||||
String city = (String) dbo.get("fangjia_city");
|
||||
String region = (String) dbo.get("fangjia_area");
|
||||
String street = (String) dbo.get("street_name");
|
||||
List<String> ps = (List<String>) dbo.get("alias");
|
||||
if(!ALL_CITY.contains(city)){
|
||||
continue;
|
||||
}
|
||||
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
|
||||
if(cityRegionMap == null){
|
||||
cityRegionMap = new HashMap<String, Map<String, Map<String, List<String>>>>();
|
||||
townDataMap.put(city, cityRegionMap);
|
||||
}
|
||||
|
||||
Map<String, Map<String, List<String>>> regoinMap = cityRegionMap.get(region);
|
||||
if(regoinMap == null){
|
||||
regoinMap = new HashMap<String, Map<String, List<String>>>();
|
||||
cityRegionMap.put(region, regoinMap);
|
||||
}
|
||||
|
||||
Map<String, List<String>> townMap = regoinMap.get("street");
|
||||
if(townMap == null){
|
||||
townMap = new HashMap<String, List<String>>();
|
||||
regoinMap.put(street, townMap);
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(ps) && ps.size() > 1) {
|
||||
Collections.sort(ps, new Comparator<String>() {
|
||||
public int compare(String arg0, String arg1) {
|
||||
if (StringUtils.isBlank(arg0)) {
|
||||
return 1;
|
||||
}
|
||||
if (StringUtils.isBlank(arg1)) {
|
||||
return -1;
|
||||
}
|
||||
return arg1.length() - arg0.length();
|
||||
}
|
||||
});
|
||||
}
|
||||
if(null != ps && !ps.contains(street)){
|
||||
ps.add(street);
|
||||
}
|
||||
townMap.put(street, ps);
|
||||
|
||||
}
|
||||
|
||||
LOGGER.info("TownData reload() size [" + townDataMap.size() + "]");
|
||||
}
|
||||
|
||||
static class ScheduleRunTask extends TimerTask {
|
||||
private int nextScheduleHour;
|
||||
@@ -174,6 +239,28 @@ public class RegionData implements PriorityOrdered {
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public static String formatStreet(String city, String region, String street) {
|
||||
if (StringUtils.isBlank(region) || StringUtils.isBlank(city) || StringUtils.isBlank(street)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String ret = null;
|
||||
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
|
||||
if(MapUtils.isNotEmpty(cityRegionMap)){
|
||||
Map<String, Map<String, List<String>>> streetMap = cityRegionMap.get(region);
|
||||
if (MapUtils.isNotEmpty(streetMap)) {
|
||||
for (Map<String, List<String>> crm : streetMap.values()) {
|
||||
if (null != ret) {
|
||||
break;
|
||||
}
|
||||
ret = format(crm, street);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public static String parseRegionFormAddress(String city, String address){
|
||||
if(StringUtils.isBlank(address)){
|
||||
return null;
|
||||
|
||||
@@ -1463,6 +1463,13 @@ public class DataUtil {
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean isNumber(String s) {
|
||||
if (s != null && !"".equals(s.trim()))
|
||||
return s.matches("^[0-9]*$");
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
public static double roundDouble(double d, int scale) {
|
||||
try {
|
||||
double ret = new BigDecimal(String.valueOf(d)).setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
|
||||
|
||||
+32
@@ -37,6 +37,8 @@ import com.fangjia.dal.mongo.open.model.EvaluateRatio;
|
||||
import com.fangjia.dal.seawater.dao.SeawaterDao;
|
||||
import com.fangjia.dal.seawater.model.Seawater;
|
||||
import com.fangjia.open.dto.*;
|
||||
import com.fangjia.open.dto.addressGovernance.StructureDto;
|
||||
import com.fangjia.open.util.CommonDtoUtils;
|
||||
import com.fangjia.service.district.*;
|
||||
import com.fangjia.service.mongo.fangjia.*;
|
||||
import com.fangjia.service.open.*;
|
||||
@@ -1085,6 +1087,21 @@ public class PropertyController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
if(model.getFloor() == null){
|
||||
JSONObject structureOb = addressMatchProxyService.addressStructure(district.getCity(), model.getAddress());
|
||||
if(structureOb != null){
|
||||
if(structureOb.containsKey("buildingNumber")){
|
||||
model.setBuildingNumber(structureOb.getString("buildingNumber"));
|
||||
}
|
||||
|
||||
if(structureOb.containsKey("roomNumber")){
|
||||
String roomNumber = structureOb.getString("roomNumber");
|
||||
model.setFloor(getFloor(roomNumber));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
|
||||
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
|
||||
if(loudong != null && loudong.getFloors() >= model.getFloor()){
|
||||
@@ -1108,6 +1125,8 @@ public class PropertyController {
|
||||
} else {
|
||||
model.setCalDate(DateUtil.getFirstDayOfWeek(new Date()));
|
||||
}
|
||||
|
||||
|
||||
Evaluation evaluation = evaluationService.evaluate(district, buildEvaluation(model, request),
|
||||
model.getCalDate());
|
||||
Double cityPriceCredit = null;
|
||||
@@ -4057,6 +4076,19 @@ public class PropertyController {
|
||||
return "fangjia.com";
|
||||
}
|
||||
|
||||
private Integer getFloor(String roomNumber){
|
||||
if(StringUtils.isNotBlank(roomNumber) && DataUtil.isNumber(roomNumber)){
|
||||
if(roomNumber.length() == 3){
|
||||
return Integer.valueOf(roomNumber.substring(0,1));
|
||||
}else if(roomNumber.length() == 4){
|
||||
return Integer.valueOf(roomNumber.substring(0,2));
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean checkExists(FacetQueryResult<NewHousePojo> houseBoList, NewHousePojo house) {
|
||||
for (NewHousePojo bo : houseBoList) {
|
||||
boolean sameName = bo.getName().equals(house.getName());
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+54
-54
@@ -40,61 +40,61 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
|
||||
index ++
|
||||
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
|
||||
def ccbMaps = null
|
||||
if(today<=10){
|
||||
ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
}else{
|
||||
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
}
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
|
||||
}
|
||||
|
||||
def allDatas = []
|
||||
allDatas << headers
|
||||
allDatas.addAll(rows)
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
// def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
// def rows = []
|
||||
//
|
||||
// def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
// //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
// //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
//
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// def today = new Date().getDate()
|
||||
//
|
||||
// println "starting 。。。。。。。。。"
|
||||
// for(def city : citys){
|
||||
//
|
||||
// index ++
|
||||
//
|
||||
//
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
//
|
||||
//
|
||||
// def ccbMaps = null
|
||||
// if(today<=10){
|
||||
// ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
// }else{
|
||||
// ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
// }
|
||||
//
|
||||
// def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
// rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
// rows.addAll(rets)
|
||||
// println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
//
|
||||
//// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// def allDatas = []
|
||||
// allDatas << headers
|
||||
// allDatas.addAll(rows)
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成->导Oracle"
|
||||
//
|
||||
// if(today > 10){
|
||||
// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
// }
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
// delPgData()
|
||||
// saveToPgSql2(datas)
|
||||
def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
delPgData()
|
||||
saveToPgSql2(datas)
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
|
||||
+1
-1
@@ -217,7 +217,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def price = ccbPriceMap.get(obj._id.toString())
|
||||
if(price){
|
||||
def mod = java.lang.Math.abs((trade.price-price)*100/price)
|
||||
if(mod<=10){
|
||||
if(mod<=20){
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
|
||||
+2
-2
@@ -392,12 +392,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202306
|
||||
def month = 202307
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202307)
|
||||
def rets = findPeGteThirdParty2(city, 202308)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+5
-5
@@ -93,13 +93,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-04-01"
|
||||
def date = "2023-05-01"
|
||||
def endDate = "2023-06-01"
|
||||
def newDate = "2023-07-01"
|
||||
def startDate = "2023-05-01"
|
||||
def date = "2023-06-01"
|
||||
def endDate = "2023-07-01"
|
||||
def newDate = "2023-08-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+1
-2
@@ -41,11 +41,10 @@ class ExportPgsqlEvaluationResult {
|
||||
exportRegionPrice(curMonth)
|
||||
|
||||
exportCommunityBasePrice(curMonth)
|
||||
// exportCommunityBasePrice1(curMonth)
|
||||
|
||||
exportCommunityPdescribeGj(curMonth)
|
||||
exportCommCaseDetailGj(curMonth)
|
||||
//
|
||||
|
||||
exportInOk()
|
||||
//
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
|
||||
+187
@@ -1,8 +1,10 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
@@ -32,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class CommonDatautils {
|
||||
|
||||
@@ -252,6 +255,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","华润佘山九里二期")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -3941,5 +3945,188 @@ class CommonDatautils {
|
||||
return files;
|
||||
}
|
||||
|
||||
static def findLogs(userId, requestUri, startDate, endDate, code){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
|
||||
if(startDate){
|
||||
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
|
||||
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
|
||||
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
|
||||
queryBuilder.must(date_range_cdate);
|
||||
}
|
||||
if(code){
|
||||
queryBuilder.must(QueryBuilders.termQuery("code", code))
|
||||
}
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
return rets.size()
|
||||
}
|
||||
|
||||
static def findLogsBySql(userId, requestUri, startDate, endDate, code){
|
||||
String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' "
|
||||
if(startDate){
|
||||
sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'"
|
||||
}
|
||||
if(code){
|
||||
sql = sql + " and code=${code}"
|
||||
}
|
||||
def total = null
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
return total;
|
||||
}
|
||||
|
||||
static def findAmapDetailBySql(city){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and status=0"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
// if(data.name.contains('华山小区G栋')){
|
||||
def point = data.location
|
||||
// def tag = null
|
||||
// if(data.get("business.keytag")){
|
||||
// tag = data.get("business.keytag")
|
||||
// }
|
||||
def business_area = null
|
||||
if(data.get("business.business_area")){
|
||||
business_area = data.get("business.business_area")
|
||||
}
|
||||
def location = [:]
|
||||
location.lon = point.x
|
||||
location.lat = point.y
|
||||
rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
|
||||
// }
|
||||
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
//用sql有些字段取不出来
|
||||
static def findDetailLogsBySql(userId, requestUri){
|
||||
String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200"
|
||||
|
||||
sql = sql + " order by c_date desc"
|
||||
def total = null
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
def ret = []
|
||||
println data.toString()
|
||||
ret << data.get("parameter_map.city")
|
||||
// ret << data.get("parameter_map.district")
|
||||
// ret << data.get("parameter_map.keyword")
|
||||
// ret << data.get("parameter_map.name")
|
||||
ret << data.get("parameter_map.address")
|
||||
// ret << data.get("parameter_map.size")
|
||||
ret << data.get("msg")
|
||||
ret << data.get("code")
|
||||
ret << data.get("c_date").toString()[0..10]
|
||||
rets << ret
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def findNewAmapDetail(city) {
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city))
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder);
|
||||
def results = []
|
||||
if(rets) {
|
||||
for (int k = 0; k < rets.size(); k++) {
|
||||
def data = rets[k];
|
||||
println data
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
static def findLogs2(userId, requestUri){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
|
||||
|
||||
queryBuilder.mustNot(QueryBuilders.termQuery("code", 200))
|
||||
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def results = []
|
||||
if(rets) {
|
||||
for (int k = 0; k < rets.size(); k++) {
|
||||
def data = rets[k];
|
||||
println data
|
||||
def ret = []
|
||||
println data.toString()
|
||||
ret << data.parameter_map.city
|
||||
ret << data.parameter_map.district
|
||||
ret << data.parameter_map.name
|
||||
ret << data.parameter_map.address
|
||||
ret << data.msg
|
||||
ret << data.code
|
||||
ret << data.c_date.toString()[0..10]
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
static def httpSaveData(def newUrl,def params){
|
||||
def header = [:]
|
||||
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
|
||||
doPost(newUrl,header, params, true, "utf-8")
|
||||
}
|
||||
|
||||
|
||||
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
try{
|
||||
URL url = new URL(targetUrl);
|
||||
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
conn.setRequestMethod("POST");
|
||||
conn.setDoOutput(true);
|
||||
conn.setUseCaches(false);
|
||||
conn.setConnectTimeout(5000);
|
||||
conn.setReadTimeout(5000);
|
||||
conn.setRequestProperty("User-Agent", UserAgents.get());
|
||||
String param = null;
|
||||
BufferedWriter mWriter = null;
|
||||
InputStreamReader inStream = null;
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
conn.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
} else {
|
||||
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
|
||||
}
|
||||
|
||||
if (isJson) {
|
||||
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
|
||||
param = JacksonUtil.beanToJson(params)
|
||||
}
|
||||
|
||||
conn.connect();
|
||||
|
||||
//println param.toString()
|
||||
//输出流
|
||||
if (StringUtils.isNotEmpty(param)) {
|
||||
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
|
||||
mWriter.write(param);
|
||||
mWriter.flush();
|
||||
mWriter.close();
|
||||
}
|
||||
if (conn.getHeaderField("Content-Encoding") != null
|
||||
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
|
||||
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
|
||||
} else {
|
||||
inStream = new InputStreamReader(conn.getInputStream(), charset);
|
||||
}
|
||||
char[] bf = new char[1024];
|
||||
int len;
|
||||
while ((len = inStream.read(bf)) != -1) {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+5
-5
@@ -38,10 +38,10 @@ class AmapAddressUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new AmapAddressUpdate()
|
||||
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
|
||||
def addr = task.formatAddress("福州", "北三环路201号")
|
||||
println addr
|
||||
// task.run(path)
|
||||
def path = "d:\\数据\\小区\\福建省小区地址与高德地图V2.xlsx"
|
||||
// def addr = task.formatAddress("福州", "北三环路201号")
|
||||
// println addr
|
||||
task.run(path)
|
||||
// task.similarMatch(path)
|
||||
// task.addressRun(path)
|
||||
// task.addressSource(path)
|
||||
@@ -52,7 +52,7 @@ class AmapAddressUpdate {
|
||||
// task.updateLocation(path)//更新经纬度
|
||||
}
|
||||
def run(path){
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"]//CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
|
||||
for (def city : citys) {
|
||||
|
||||
+36
-1
@@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def ret = createNewDatas("福州")
|
||||
saveToDB(ret)
|
||||
|
||||
def results = selectLastResultDatas(ret);
|
||||
|
||||
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
}
|
||||
def sb = appendResult(headers, ret)
|
||||
def sb = appendResult(headers, results)
|
||||
DataHandleUtils.outPutFile(fileName, sb, charset);
|
||||
FzFtpUtils.ftpUpload(fileName)
|
||||
//生成加密数据文件并上传
|
||||
FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset);
|
||||
}
|
||||
|
||||
def selectLastResultDatas(def rets){
|
||||
def rows = []
|
||||
for(def r : rets){
|
||||
rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]]
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def appendResult(def headers, def ret){
|
||||
@@ -147,6 +157,29 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
m << 0
|
||||
}
|
||||
m << curDate.toInteger();
|
||||
if(curPrice && dbo){
|
||||
if(dbo.value_type == 1){
|
||||
m << "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
m << "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
m << "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
m << "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
m << "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
m << "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
m << "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
m << "无方式"
|
||||
}else{
|
||||
m << ""
|
||||
}
|
||||
}else{
|
||||
m << "无"
|
||||
}
|
||||
result << m
|
||||
}
|
||||
return result
|
||||
@@ -300,6 +333,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
//按comm_id修改,小区名重新赋值,防止区域和小区名修改
|
||||
udata.put("region", r[3])
|
||||
udata.put("name", r[4])
|
||||
udata.put("value_type", r[7])
|
||||
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def o = new BasicDBObject()
|
||||
@@ -312,6 +346,7 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
o.put("avg_price", r[5].toInteger())
|
||||
o.put("month", r[6].toInteger())
|
||||
o.put("c_date", new Date())
|
||||
o.put("value_type", r[7])
|
||||
FZ_DISTRICT_STAT_COLL.save(o)
|
||||
}
|
||||
}
|
||||
|
||||
+6
-2
@@ -140,6 +140,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def key = dist.region +"##" + dist.name
|
||||
|
||||
def weekendPriceObj = weekendPriceMap.get(key)
|
||||
if(weekendPriceObj){
|
||||
def curPriceObj = curMonthlyPriceMap.get(key)
|
||||
@@ -155,7 +156,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
if(curPriceObj){
|
||||
curPrice = curPriceObj.avg_price
|
||||
}else{
|
||||
curPrice = weekendPriceObj.avg_price
|
||||
curPrice = weekendPriceObj.avg_price
|
||||
}
|
||||
|
||||
if(weekendPriceObj.value_type!=1){
|
||||
@@ -197,7 +198,9 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
data.put("commid", dist._id.toString())
|
||||
data.put("mom", mom)
|
||||
data.put("yoy", yoy)
|
||||
data.put("avg_price", curPrice)
|
||||
data.put("avg_price", weekendPriceObj.avg_price)
|
||||
data.put("value_type", weekendPriceObj.value_type)
|
||||
data.put("credit", weekendPriceObj.credit)
|
||||
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
|
||||
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
|
||||
}
|
||||
@@ -209,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
data.put("estate_type", dist.estate_type2)
|
||||
data.put("date", curMonth)
|
||||
data.put("avg_price", curPrice)
|
||||
data.put("value_type", weekendPriceObj.value_type)
|
||||
data.put("credit", weekendPriceObj.credit)
|
||||
data.put("m_date", new Date())
|
||||
data.put("commid", dist._id.toString())
|
||||
|
||||
+35
-27
@@ -51,59 +51,61 @@ class FzDataResultStat {
|
||||
})
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
||||
def idMaps = [:]
|
||||
for(def d : dists){
|
||||
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
|
||||
}
|
||||
|
||||
def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")))
|
||||
def distkeys = []
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
for(def obj : list1){
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
if(!distkeys.contains(key)){
|
||||
distkeys << key
|
||||
}else{
|
||||
println "重复:"+key
|
||||
}
|
||||
map1.put(key, row)
|
||||
map1.put(key, obj)
|
||||
}
|
||||
|
||||
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
def list2 = findFzDistrictStat(lastMonth)
|
||||
def map2 = [:]
|
||||
for(def s : list2) {
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
map2.put(key, row)
|
||||
for(def obj : list2) {
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
map2.put(key, obj)
|
||||
}
|
||||
def rows = []
|
||||
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||
map1.each {k,list->
|
||||
def row = []
|
||||
def index = 0
|
||||
map1.each {k,obj1->
|
||||
def keys = k.split("##")
|
||||
|
||||
def lastObj = map2.get(k)
|
||||
def lastPrice = null
|
||||
if(lastObj){
|
||||
lastPrice = lastObj[5]
|
||||
}
|
||||
def lastPrice = map2.get(k)
|
||||
|
||||
def markup = null
|
||||
try{
|
||||
if(list && lastPrice && lastPrice.toInteger()>0){
|
||||
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
|
||||
if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){
|
||||
markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger()
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.findDistrictById(list[1])
|
||||
def flag = "有效"
|
||||
|
||||
if(dist && dist.status!=0){
|
||||
flag = "失效"
|
||||
def flag = "无效"
|
||||
if(obj1){
|
||||
flag = idMaps.get(obj1.comm_id)
|
||||
}
|
||||
|
||||
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
|
||||
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag]
|
||||
rows << row
|
||||
index++
|
||||
println row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
@@ -255,4 +257,10 @@ class FzDataResultStat {
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
def findFzDistrictStat(lastMonth){
|
||||
def q = new BasicDBObject()
|
||||
q.put("month", lastMonth)
|
||||
return FZ_DISTRICT_STAT_COLL.find(q).toList()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+3
-3
@@ -76,7 +76,7 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
row << dist.name
|
||||
row << o.s_date
|
||||
|
||||
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
|
||||
def zgf_price = getLastThirdPrice(o.xingfuli_esf_price, curPrice)
|
||||
row << zgf_price
|
||||
if(zgf_price){
|
||||
add = true;
|
||||
@@ -217,8 +217,8 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.s_date){
|
||||
retMap.s_date = curDate
|
||||
}
|
||||
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
|
||||
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
|
||||
if(!retMap.xingfuli_esf_price && d.xingfuli_esf_price){
|
||||
retMap.xingfuli_esf_price = d.xingfuli_esf_price
|
||||
}
|
||||
if(!retMap.anjuke_esf_price && d.get("anjuke_esf_price")){
|
||||
retMap.anjuke_esf_price = d.get("anjuke_esf_price")
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-06-01"
|
||||
static def statDate = "2023-07-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
@@ -90,4 +90,53 @@ class FileUtils {
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){
|
||||
File file = new File(fileName);
|
||||
// if (file.exists()) {
|
||||
// file.delete();
|
||||
// }
|
||||
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
if(headers){
|
||||
sb.append(headers.join(splitChar))
|
||||
sb.append("\n");
|
||||
}
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
if(row){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), splitChar))
|
||||
sb.append("\n");
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
|
||||
+44
-7
@@ -1,19 +1,19 @@
|
||||
package com.fangjia.script.gj
|
||||
package com.fangjia.script.gj.city
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class CityStatExport {
|
||||
|
||||
@@ -28,8 +28,33 @@ class CityStatExport {
|
||||
def task = new CityStatExport();
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.regionStat()
|
||||
// task.regionStat()
|
||||
// task.stat()
|
||||
// task.houseStat()
|
||||
task.addFieldStat()
|
||||
}
|
||||
|
||||
def addFieldStat(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d ->
|
||||
def cityInfoDto = CommonDatautils.findCityInfo(d['B'])
|
||||
if(cityInfoDto){
|
||||
def detailMap = cityInfoDto.district_detail
|
||||
d['D'] = detailMap.address.toDouble()*100
|
||||
d['E'] = 100
|
||||
d['F'] = cityInfoDto.price_coverage.toDouble()*100
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def houseStat(){
|
||||
def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def houseCount = findHouseCount(d['F'], 20230201)
|
||||
// d['H'] = houseCount
|
||||
d['J'] = findHouse(d['F'])
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def stat(){
|
||||
@@ -47,7 +72,7 @@ class CityStatExport {
|
||||
// def citys = CommonDatautils.findAllCity();
|
||||
|
||||
|
||||
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"]
|
||||
def ret = []
|
||||
ret << ["城市","小区总量"]
|
||||
// ret << ["城市","楼栋总量", "房号总量"]
|
||||
@@ -78,7 +103,7 @@ class CityStatExport {
|
||||
ret << m
|
||||
println m
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
|
||||
ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3")
|
||||
|
||||
}
|
||||
|
||||
@@ -124,7 +149,7 @@ class CityStatExport {
|
||||
def findHouseCount(def city, def startDate){
|
||||
def total = 0
|
||||
try{
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
|
||||
String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
@@ -136,6 +161,18 @@ class CityStatExport {
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
def findHouse(city) {
|
||||
int startDate = 20230201
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
|
||||
.must(date_range)
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
return houses.size()
|
||||
}
|
||||
|
||||
def findLoudongCount(String city) {
|
||||
return LOUDONG.count(new BasicDBObject('city', city))
|
||||
}
|
||||
+28
@@ -172,4 +172,32 @@ class FangjiaClientUtils {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1"
|
||||
if(floor){
|
||||
url = url + "&floor="+floor.toString().toDouble().toInteger()
|
||||
}
|
||||
if(maxFloor){
|
||||
url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger()
|
||||
}
|
||||
if(room){
|
||||
url = url + "&room="+room.toString().toDouble().toInteger()
|
||||
}
|
||||
if(hall){
|
||||
url = url + "&hall="+hall.toString().toDouble().toInteger()
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+33
-4
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
// strandField(offices)
|
||||
// }
|
||||
|
||||
|
||||
|
||||
// addBlock()
|
||||
|
||||
// addFieldValue()
|
||||
|
||||
// updateErrorField()
|
||||
export()
|
||||
test()
|
||||
// export()
|
||||
// test()
|
||||
// addOffice()
|
||||
fileAddBlock()
|
||||
}
|
||||
|
||||
def export(){
|
||||
@@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
|
||||
def fileAddBlock(){
|
||||
|
||||
def rets = findAllBlockRange()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d->
|
||||
def location = d['T']
|
||||
if(location && !d['F']){
|
||||
def arrs = location.split(",")
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
|
||||
if(baidu){
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
|
||||
def lastObj = null
|
||||
for (def r : rets) {
|
||||
def flag = InsidePolygon.isInArea(lng2, lat2, r.points)
|
||||
if (flag && r.name) {
|
||||
lastObj = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lastObj) {
|
||||
d['F'] = lastObj.name
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
def datas = findHouseCount("北京", 20230201, 20230430)
|
||||
ExcelUtils.write("D:\\数据\\天弘基金.xlsx", datas, "样例")
|
||||
|
||||
|
||||
|
||||
def findHouseCount(def city, def startDate, def endDate){
|
||||
def rets = []
|
||||
try{
|
||||
// String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='ke.com' order by p_date desc"
|
||||
def startMdate = DateUtils.parseFromTightByPattern(startDate.toString(),"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
|
||||
rets << ["城市","区域","板块","名称","面积","单价","总价","室","厅","卫","装修","朝向","发布时间","物业类型","楼层状况","楼高","建筑结构","建筑年代","标题","URL"]
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
// QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(QueryBuilders.termQuery("root", "ke.com"))
|
||||
.must(date_range).must(area_range)//.must(date_range_mdate)
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses) {
|
||||
|
||||
for (int k = 0; k < houses.size(); k++) {
|
||||
println k
|
||||
def data = houses[k];
|
||||
def r = []
|
||||
r << data.city
|
||||
r << data.region
|
||||
r << data.block
|
||||
r << data.district_name
|
||||
r << data.area
|
||||
r << data.avg_price
|
||||
r << data.total
|
||||
r << data.room
|
||||
r << data.hall
|
||||
r << data.toilet
|
||||
r << data.fitment
|
||||
r << data.direction
|
||||
r << data.p_date
|
||||
r << data.estate_type
|
||||
r << data.floor_info
|
||||
r << data.height
|
||||
r << data.build_structure
|
||||
r << data.build_date
|
||||
r << data.name
|
||||
r << data.url
|
||||
rets << r
|
||||
if(rets.size()>500){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return rets
|
||||
}
|
||||
+2
-2
@@ -25,8 +25,8 @@ class DistrictLocationBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationBatch()
|
||||
// task.run()
|
||||
task.deleteDist()
|
||||
task.run()
|
||||
// task.deleteDist()
|
||||
}
|
||||
|
||||
def deleteDist(){
|
||||
|
||||
+49
-21
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
WebDriver driver = null;
|
||||
|
||||
//广州,佛山 重庆
|
||||
static main(args) {
|
||||
@@ -50,7 +50,7 @@ public class BeiKeCrawlByChrome {
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver = new ChromeDriver()
|
||||
def citys = getAllCityUrls(driver)
|
||||
|
||||
driver = getLogin(driver, "https://sh.ke.com/")
|
||||
@@ -93,19 +93,28 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
try{
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
println "异常Url:"+url
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
return driver;
|
||||
}
|
||||
|
||||
def crawlData(driver, citys){
|
||||
@@ -130,7 +139,8 @@ public class BeiKeCrawlByChrome {
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
driver = checkDocument(driver, newUrl);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -142,8 +152,9 @@ public class BeiKeCrawlByChrome {
|
||||
regionText = "昆山"
|
||||
}
|
||||
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println url +" 详情链接:"+detailUrls.size()
|
||||
def map = getPageDeail(driver, newCity, href, regionText)
|
||||
driver = map.driver
|
||||
println url +" 详情链接:"+map.detailUrls.size()
|
||||
}
|
||||
}
|
||||
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
|
||||
@@ -191,9 +202,9 @@ public class BeiKeCrawlByChrome {
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
// def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = checkDocument(driver, url);
|
||||
driver = checkDocument(driver, url);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
@@ -203,10 +214,11 @@ public class BeiKeCrawlByChrome {
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
println "开始查库 "+new Date().toString()
|
||||
def dbo = findTrade(durl , "贝壳找房")
|
||||
|
||||
println "结束查库"+new Date().toString()
|
||||
if (!dbo) {
|
||||
parse(city, e, region, driver, null)
|
||||
driver = parse(city, e, region, driver, null)
|
||||
}
|
||||
// else if(dbo && !dbo.lift){
|
||||
// parse(city, e, region, driver, dbo._id)
|
||||
@@ -225,7 +237,10 @@ public class BeiKeCrawlByChrome {
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
def m = [:]
|
||||
m.driver = driver
|
||||
m.detailUrls = detailUrls
|
||||
return m
|
||||
}
|
||||
|
||||
def parse(city,e, region, driver, id){
|
||||
@@ -233,8 +248,10 @@ public class BeiKeCrawlByChrome {
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
driver = checkDocument(driver, detailUrl);
|
||||
def detailDoc = Jsoup.parse(driver.getPageSource())
|
||||
|
||||
println "打开获取页面 "+new Date().toString()
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -335,18 +352,21 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println "解析页面完成 "+new Date().toString()
|
||||
println detail
|
||||
if(id){
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
|
||||
}else{
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
println "保存修改完成 "+new Date().toString()
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
return driver
|
||||
}
|
||||
|
||||
|
||||
@@ -478,7 +498,7 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def getLogin(driver, url){
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
|
||||
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
@@ -494,17 +514,25 @@ public class BeiKeCrawlByChrome {
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("17148854005");
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
// elemUsername.sendKeys("16283166371");
|
||||
// elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
|
||||
return driver
|
||||
}
|
||||
|
||||
|
||||
+57
-36
@@ -34,6 +34,7 @@ public class LianjiaCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
WebDriver driver = null;
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -64,7 +65,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run1(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
@@ -122,7 +123,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
// cityMap = cityMap.reverse()
|
||||
driver = getLogin(driver, "https://sh.lianjia.com/")
|
||||
@@ -206,17 +207,11 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
def cityUrl = ret[1]
|
||||
def newUrl = cityUrl+"chengjiao/"
|
||||
try{
|
||||
driver.get(newUrl)
|
||||
}catch(Exception e){
|
||||
println newUrl +" 链接有问题!"
|
||||
continue
|
||||
}
|
||||
|
||||
// driver.get(newUrl)
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
|
||||
// Thread.sleep(10000)
|
||||
driver = checkDocument(driver, newUrl);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -226,8 +221,9 @@ public class LianjiaCrawlByChrome {
|
||||
regionText = "昆山"
|
||||
}
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, newCity, href, regionText)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
def map = getPageDeail(driver, newCity, href, regionText)
|
||||
driver = map.driver
|
||||
println newUrl +" 详情链接:"+map.detailUrls.size()
|
||||
}
|
||||
}
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
@@ -267,30 +263,41 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
driver.get(url)
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
try{
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}catch(Exception e){
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
return driver;
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl, regionText){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "pg${page}/"
|
||||
driver.get(url)
|
||||
// driver.get(url)
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def doc = checkDocument(driver, url);
|
||||
driver = checkDocument(driver, url);
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
@@ -303,7 +310,7 @@ public class LianjiaCrawlByChrome {
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver, null)
|
||||
driver = parse(city, e, regionText, driver, null)
|
||||
}
|
||||
// else if(dbo && !dbo.lift){
|
||||
// parse(city, e, regionText, driver, dbo._id)
|
||||
@@ -326,7 +333,10 @@ public class LianjiaCrawlByChrome {
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
def m = [:]
|
||||
m.driver = driver
|
||||
m.detailUrls = detailUrls
|
||||
return m
|
||||
}
|
||||
|
||||
def parse(city,e, regionText, driver, id){
|
||||
@@ -334,8 +344,11 @@ public class LianjiaCrawlByChrome {
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if(price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
|
||||
// driver.get(detailUrl)
|
||||
driver = checkDocument(driver, detailUrl);
|
||||
def detailDoc = Jsoup.parse(driver.getPageSource())
|
||||
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -450,6 +463,7 @@ public class LianjiaCrawlByChrome {
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
return driver;
|
||||
}
|
||||
|
||||
|
||||
@@ -567,11 +581,11 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityList = []
|
||||
def citys= ["福州"]
|
||||
def citys= ["衢州"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityList << [e.text(), href]
|
||||
}
|
||||
}
|
||||
@@ -593,16 +607,23 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
elemUsername.sendKeys("17148854005");
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
// elemUsername.sendKeys("17740818975");
|
||||
// elemPassword.sendKeys("13qeadzc");
|
||||
|
||||
// elemUsername.sendKeys("16283166371");
|
||||
// elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(15000)
|
||||
Thread.sleep(20000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(10000)
|
||||
driver.close()
|
||||
driver = new ChromeDriver()
|
||||
Thread.sleep(15000)
|
||||
driver = getLogin(driver, url)
|
||||
println "异常Url:"+url
|
||||
}
|
||||
return driver
|
||||
}
|
||||
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
package com.fangjia.script.gj.township
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class StreetProcess {
|
||||
|
||||
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
|
||||
|
||||
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
|
||||
|
||||
|
||||
final DBCollection POLYLINE235 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new StreetProcess();
|
||||
// println RegionData.formatRegion("鞍山","铁东区")
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
|
||||
def region = RegionData.formatRegion("三明市", "三元区")
|
||||
println region
|
||||
|
||||
// standard()
|
||||
// matchStreet()
|
||||
// synchTownship()
|
||||
}
|
||||
|
||||
def synchTownship(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city_name","福州市")
|
||||
q.put("shape",new BasicDBObject('$exists', true))
|
||||
def dbcursor = TOWNSHIP.find(q)
|
||||
for(def d : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", d.shape)
|
||||
TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
def matchStreet(){
|
||||
def polylineMap = findStreetPolyline()
|
||||
def q = new BasicDBObject()
|
||||
// q.put("fangjia_city", "福州")
|
||||
// q.put("shape",new BasicDBObject('$exists', false))
|
||||
def dbcursor = TOWNSHIP.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def streetMap = polylineMap.get(d.fangjia_city+"##"+d.fangjia_area)
|
||||
if(streetMap){
|
||||
def alias = d.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(d.street_name)){
|
||||
alias << d.street_name
|
||||
}
|
||||
for(def streetName : alias){
|
||||
def polyline = streetMap.get(streetName)
|
||||
if(polyline){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
TOWNSHIP.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
println index + " "+ d.get("_id")
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findStreetPolyline(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("fjw_city","福州")
|
||||
def dbcursor = POLYLINE235.find(q)
|
||||
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.fjw_city+"##"+d.fjw_region
|
||||
def streetMap = map.get(key)
|
||||
if(!streetMap){
|
||||
streetMap = [:]
|
||||
map.put(key, streetMap)
|
||||
}
|
||||
streetMap.put(d.town, d.polyline)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def standard(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("fjw_city",new BasicDBObject('$exists', false))
|
||||
def dbcursor = POLYLINE235.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index ++
|
||||
def city = RegionData.formatCity(d.city)
|
||||
def region = RegionData.formatRegion(d.city, d.region)
|
||||
// def street = null
|
||||
// if(city && region){
|
||||
// street = RegionData.formatStreet(city, region, d.town)
|
||||
// }
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fjw_city", city)
|
||||
udata.put("fjw_region", region)
|
||||
// if(street){
|
||||
// udata.put("fjw_street", street)
|
||||
// }
|
||||
POLYLINE235.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def standard1(){
|
||||
def list = TOWNSHIP.find(new BasicDBObject()).toList()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(list.size())
|
||||
for(def i=0;i<list.size();i++){
|
||||
final def d = list[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def city = RegionData.formatCity(d.city_name)
|
||||
if(city){
|
||||
def fangjia_province = ProvinceData.getProvince(city)
|
||||
def region = RegionData.formatRegion(city, d.area_name)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fangjia_city", city)
|
||||
udata.put("fangjia_area", region)
|
||||
udata.put("fangjia_province", fangjia_province)
|
||||
TOWNSHIP.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println totalCount.get()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
}
|
||||
+372
@@ -0,0 +1,372 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 厦门银行
|
||||
* 福建省,重庆市poi数据处理
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
|
||||
public static def version = new Date().getTime()
|
||||
|
||||
public static def TAGS = ["加油站","其它能源站","加气站","洗车场","充电站","餐饮相关场所","休闲餐饮场所","购物相关场所","商场","超级市场","家电电子卖场","花鸟鱼虫市场","家居建材市场","综合市场","特色商业街","生活服务场所","邮局","事务所","人才市场","洗浴推拿场所","体育休闲服务场所","运动场馆","娱乐场所","度假疗养场所","休闲场所","医疗保健服务场所","综合医院","专科医院","诊所","急救中心","疾病预防机构","住宿服务相关","宾馆酒店","旅馆招待所","风景名胜相关","公园广场","风景名胜","产业园区","政府及社会团体相关","政府机关","外国机构","社会团体","公检法机构","交通车辆管理","工商税务机构","科教文化场所","博物馆","展览馆","会展中心","美术馆","图书馆","科技馆","天文馆","文化宫","档案馆","文艺团体","传媒机构","学校","科研机构","培训机构","驾校","机场相关","银行","银行相关","保险公司","证券公司","财务公司","公司企业","知名企业","公司","工厂","农林牧渔基地","收费站","服务区","标志性建筑物","地名地址信息","热点地名","紧急避难场所"]
|
||||
|
||||
|
||||
public final static DBCollection COMPANY_206_COLL = MongoUtils.getCollectionByName("business", "company", "192.168.0.206",27020)
|
||||
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
|
||||
|
||||
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GaoDePoiDataProcess(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// synchMapPoi();
|
||||
synchCompany()
|
||||
}
|
||||
|
||||
def synchCompany(){
|
||||
|
||||
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
|
||||
def citys = ["重庆市"]
|
||||
for(def c : citys){
|
||||
def townMap = findTownRange(c)
|
||||
def companyDatas = findCompanyDatas(c)
|
||||
println c+ " 加载公司数据:"+companyDatas.size()
|
||||
def rows = running(companyDatas, "company", townMap)
|
||||
save(c, rows)
|
||||
println "完成"
|
||||
}
|
||||
}
|
||||
|
||||
def findCompanyDatas(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("done", 1)
|
||||
q.put("geo_city", city)
|
||||
q.put("geo_level", new BasicDBObject('$in', ['门牌号','兴趣点']))
|
||||
return COMPANY_206_COLL.find(q).toList()
|
||||
}
|
||||
|
||||
def synchMapPoi(){
|
||||
//"三明市","漳州市","厦门市","福州市"
|
||||
//三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"
|
||||
def citys = ["福州市","重庆市"]
|
||||
for(def city : citys){
|
||||
def results = CommonDatautils.findAmapDetailBySql(city)
|
||||
println results.size()
|
||||
def rows = running(results, "map")
|
||||
save(city, rows)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def running(results, type, townMap){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(type == "map"){
|
||||
rets.addAll(createData(map, townMap))
|
||||
}else if(type == "company"){
|
||||
rets.addAll(createCompanyData(map, townMap))
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createCompanyData(map, townMap){
|
||||
def retMap = [:]
|
||||
retMap.id = new ObjectId().toString()
|
||||
retMap.fjw_province = map.province_name
|
||||
retMap.fjw_city = map.city_name
|
||||
|
||||
def fjw_region = RegionData.formatRegion(map.city_name, map.geo_district)
|
||||
if(fjw_region){
|
||||
retMap.fjw_district = fjw_region
|
||||
}
|
||||
|
||||
if(!retMap.fjw_district ){
|
||||
retMap.fjw_district = RegionData.formatRegion(map.city_name, map.region_name)
|
||||
}
|
||||
retMap.province = map.geo_province
|
||||
retMap.city = map.geo_city
|
||||
retMap.district = map.geo_district
|
||||
retMap.cat = "企业"
|
||||
retMap.name = map.ent_name
|
||||
// def pointList = townMap.get(retMap.fjw_district)
|
||||
|
||||
def locationStr = map.geo_location
|
||||
if(locationStr){
|
||||
def arrs = locationStr.split(",")
|
||||
|
||||
retMap.township = InsidePolygon.isInArea2(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), townMap)
|
||||
|
||||
def gd_location = [:]
|
||||
gd_location.lon = arrs[0].toString().toDouble()
|
||||
gd_location.lat = arrs[1].toString().toDouble()
|
||||
retMap.location = gd_location
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = baiduMap.lng
|
||||
location.lat = baiduMap.lat
|
||||
retMap.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = gpsLoactionMap.lng
|
||||
gpslocation.lat = gpsLoactionMap.lat
|
||||
retMap.gps_location = gpslocation
|
||||
}
|
||||
|
||||
if(!retMap.township){
|
||||
def gaodeMap = getTownshipInfo(locationStr)
|
||||
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
retMap.township = gaodeMap.township
|
||||
}
|
||||
|
||||
if(!retMap.fjw_district){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district)
|
||||
}
|
||||
}
|
||||
if(!fjw_region && !retMap.fjw_district){
|
||||
println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap
|
||||
}
|
||||
retMap.fjw_district = fjw_region
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
retMap.address = map.reg_addr
|
||||
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.ent_name, " ")
|
||||
retMap.version = version
|
||||
retMap.status = 1
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def createData(map, townMap){
|
||||
def fjw_province = null
|
||||
if(map.city.contains("重庆")){
|
||||
fjw_province = "重庆"
|
||||
}else{
|
||||
fjw_province = "福建"
|
||||
}
|
||||
def fjw_city = RegionData.formatCity(map.city)
|
||||
def fjw_region = RegionData.formatRegion(fjw_city, map.region)
|
||||
map.id = new ObjectId().toString()
|
||||
map.fjw_province = fjw_province
|
||||
map.fjw_city = fjw_city
|
||||
|
||||
if(map.cat){
|
||||
def arrs = map.cat.split(";")
|
||||
map.type = arrs[0]
|
||||
if(arrs.size()>=2){
|
||||
map.secondary_type = arrs[1]
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
map.small_type = arrs[2]
|
||||
}
|
||||
}
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = baiduMap.lng
|
||||
location.lat = baiduMap.lat
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = gpsLoactionMap.lng
|
||||
gpslocation.lat = gpsLoactionMap.lat
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
|
||||
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
map.township = gaodeMap.township
|
||||
}
|
||||
|
||||
if(!fjw_region){
|
||||
if(gaodeMap && gaodeMap.district){
|
||||
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
}
|
||||
}
|
||||
if(!fjw_region){
|
||||
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
|
||||
}
|
||||
map.fjw_district = fjw_region
|
||||
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
|
||||
map.version = version
|
||||
|
||||
if(map.secondary_type && TAGS.contains(map.secondary_type)){
|
||||
map.status = 0
|
||||
}else{
|
||||
map.status = 1
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def save(city,rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient("192.168.0.12")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "[${city}] 结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.district = district
|
||||
map.township = township
|
||||
return map;
|
||||
}
|
||||
|
||||
def findTownRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
// def regionMap = townMap.get(d.fjw_region)
|
||||
// if(!regionMap){
|
||||
// regionMap = [:]
|
||||
// townMap.put(d.fjw_region, regionMap)
|
||||
// }
|
||||
// def townInfoMap = regionMap.get(d.town)
|
||||
// if(townInfoMap){
|
||||
// townInfoMap = [:]
|
||||
// regionMap.put()
|
||||
// }
|
||||
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class NetworkDistanceBatch extends BaseBatch{
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new NetworkDistanceBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def networks = ExcelUtils.read("D:\\数据\\厦门\\网点.xlsx", 0, 1)
|
||||
def citys = ["厦门","三明","宁德","泉州","福州","漳州","南平","龙岩","莆田","重庆"]
|
||||
for(def c : citys){
|
||||
run(c, "district", networks)
|
||||
}
|
||||
}
|
||||
|
||||
def run(city, cat, networks){
|
||||
def datas = findDistrictData(city, cat, ["city","region", "name", "category", "location"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def dist = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def map = createData(dist, networks)
|
||||
if(map){
|
||||
rets << map
|
||||
}
|
||||
// println city +" "+cat +" "+totalCount.get() +"/"+datas.size()+" "+dist.name+" "+dist._id
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
saveDatas(rets);
|
||||
println cat +" "+city +" 总量:"+datas.size()+" 在5公里内总量:"+rets.size()
|
||||
}
|
||||
|
||||
def saveDatas(def rets){
|
||||
NETWORK_COLL.insert(rets)
|
||||
}
|
||||
|
||||
def createData(dist, networks){
|
||||
|
||||
def lastRest = null
|
||||
def minDistance = Integer.MAX_VALUE
|
||||
for(def row : networks){
|
||||
def locationstr = row['D']
|
||||
if(locationstr){
|
||||
def arrs = locationstr.split(",")
|
||||
def distance = GeoUtils.calculateDistance(dist.location.lon, dist.location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
|
||||
if(distance <= minDistance){
|
||||
minDistance = distance
|
||||
lastRest = row
|
||||
}
|
||||
}
|
||||
}
|
||||
if(minDistance<=5000){
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("_id", dist._id);
|
||||
obj.put("city", dist.city);
|
||||
obj.put("region", dist.region);
|
||||
obj.put("name", dist.name);
|
||||
obj.put("distance", (int)minDistance);
|
||||
obj.put("network", lastRest['B'])
|
||||
return obj
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def findDistrictData(def city, cat, fields){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status", 0);
|
||||
q.put("category", cat);
|
||||
q.put("city", city);
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
for(def field : fields){
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
|
||||
return SEAWEED_CONN.find(q, keys).toList();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class PoiAndSeaweedExportCSV extends BaseBatch{
|
||||
|
||||
static main(args) {
|
||||
runBatch(new PoiAndSeaweedExportCSV(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
|
||||
|
||||
poiExport(citys)
|
||||
|
||||
}
|
||||
|
||||
def poiExport(citys){
|
||||
def fileName = "/home/goojia/xiamen/BIGDATA_POI.csv"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\厦门\\BIGDATA_POI.csv"
|
||||
}
|
||||
|
||||
for(def c : citys){
|
||||
def rets = CommonDatautils.findNewAmapDetail(c)
|
||||
FileUtils.batchCreateFileBySplitChar(rets, fileName, null, "utf-8", true, "@|@");
|
||||
}
|
||||
}
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
|
||||
class SeaweedDataSynch {
|
||||
|
||||
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new SeaweedDataSynch()
|
||||
task.run()
|
||||
// println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index++
|
||||
updating(c, "district")
|
||||
updating(c, "office")
|
||||
updating(c, "shop")
|
||||
println index +" "+ c + " 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def map = createData(dist)
|
||||
|
||||
//往100 上同步
|
||||
updateData(map)
|
||||
|
||||
//往12 es上同步
|
||||
// def newUrl = targetUrl +"_doc/"+map.id
|
||||
// CommonDatautils.httpSaveData(newUrl, map)
|
||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println dist.city +" "+dist.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def updateData(map){
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(k!="_id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
SEAWEED_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
|
||||
}
|
||||
|
||||
def createData(dist){
|
||||
def map = [:]
|
||||
map._id = dist._id
|
||||
map.city = dist.city
|
||||
map.region = dist.region
|
||||
map.name = dist.name
|
||||
map.address = dist.address
|
||||
map.block = dist.block
|
||||
map.alias= dist.alias
|
||||
map.addr_alias = dist.addr_alias
|
||||
map.category = dist.cat
|
||||
map.estate_type = dist.estate_type2
|
||||
map.status = dist.status
|
||||
map.township = dist.township
|
||||
map.finish_build_time = dist.create_year?dist.create_year.toString():null
|
||||
if(dist.lng){
|
||||
def loc = [:]
|
||||
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
|
||||
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
|
||||
map.location = loc
|
||||
}
|
||||
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
|
||||
map.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
|
||||
map.gps_location = gpslocation
|
||||
}
|
||||
|
||||
map.build_type = dist.build_type
|
||||
map.households = dist.household_count
|
||||
map.committee = dist.committee
|
||||
if(dist.estate_charge && dist.estate_charge.toString().isNumber()){
|
||||
def fee = dist.estate_charge.toString().toFloat()
|
||||
if(fee < 100){
|
||||
map.property_fee = DataHandleUtils.getDoubleRound(2, fee)
|
||||
}
|
||||
}
|
||||
|
||||
if(dist.car_site_count && dist.car_site_count.toString().isNumber()){
|
||||
map.car_site_count = dist.car_site_count.toString().toInteger()
|
||||
}
|
||||
if(dist.property_life && dist.property_life.toString().isNumber()){
|
||||
map.property_life = dist.property_life.toString().toInteger()
|
||||
}
|
||||
|
||||
map.house_property = dist.house_property
|
||||
|
||||
if(dist.build_area && dist.build_area.toString().isNumber()){
|
||||
map.build_area = dist.build_area.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
if(dist.all_area && dist.all_area.toString().isNumber()){
|
||||
map.floor_area = dist.all_area.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
map.build_structure = dist.build_structure
|
||||
map.developer = dist.developer
|
||||
map.property_company = dist.estate_company
|
||||
map.plot_ratio = dist.cubage_percent
|
||||
map.landscaping_ratio = dist.virescence_percent
|
||||
if(dist.loudong_count && dist.loudong_count.toString().isNumber()){
|
||||
map.building_count = dist.loudong_count.toString().toInteger()
|
||||
}
|
||||
|
||||
map.schools = dist.schools
|
||||
map.main_house_type = dist.main_type
|
||||
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
|
||||
map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat())
|
||||
}
|
||||
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
return map
|
||||
}
|
||||
}
|
||||
+21
@@ -272,4 +272,25 @@ class InsidePolygon {
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
public static def isInArea2(lng, lat, pointMap){
|
||||
def flag = false;
|
||||
Set<String> keys = pointMap.keySet();
|
||||
def name = null
|
||||
for(String key :keys){
|
||||
def prints = pointMap.get(key)
|
||||
for(def pointArr : prints){
|
||||
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
|
||||
if(flag){
|
||||
def arrs = key.split("##")
|
||||
name = arrs[1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(flag){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -64,7 +64,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static {
|
||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
@@ -92,7 +92,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "开始加载【${city}】"
|
||||
// if(city == "石家庄"){
|
||||
// if(city == "福州"){
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
|
||||
+2
-2
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
|
||||
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
|
||||
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产","幸福里APP":"幸福里"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDistrictPrice(), args);
|
||||
@@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
def run1(){
|
||||
|
||||
println "starting......"
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), 0), "yyyy-MM-dd")
|
||||
def citys = CommonUtils.findAllCity()
|
||||
esSources.each{esSource, mgSource->
|
||||
for(def city : citys){
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf"
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
|
||||
]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.Document
|
||||
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]]
|
||||
while (startDate.before(endDate)) {
|
||||
calendar.add(Calendar.DATE, 1)
|
||||
def nextDate = calendar.getTime()
|
||||
count++
|
||||
///property/addressEvaluate /address/financialRating /v2/property/detail
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
// if (!hashSet.contains(result)) {
|
||||
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
// hashSet << result
|
||||
// } else {
|
||||
// println("已存在")
|
||||
// }
|
||||
}
|
||||
if (count % 7 == 0) {
|
||||
hashSet.clear()
|
||||
}
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情")
|
||||
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390"
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
|
||||
|
||||
/**
|
||||
* 冪帠僅蛌華硊<br>
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package com.fangjia.script.zsbank
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class ZheShangBank {
|
||||
static main(args) {
|
||||
def task = new ZheShangBank()
|
||||
// task.run()
|
||||
// task.detail()
|
||||
task.sample()
|
||||
}
|
||||
|
||||
def sample(){
|
||||
def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt")
|
||||
def rets = []
|
||||
rets << ["时间","Msg", "城市", "区域", "名称","月份"]
|
||||
def map = [:]
|
||||
for(def line : runedCitys){
|
||||
def ret = []
|
||||
def date = line.substring(line.indexOf("log.")+4, line.indexOf(":"))
|
||||
def msg = line.substring(line.lastIndexOf("error :")+7, line.size())
|
||||
def arrs = msg.split(" ")
|
||||
ret << date
|
||||
for(def i=0;i<arrs.size();i++){
|
||||
def s = arrs[i]
|
||||
if(i == 0){
|
||||
ret << s
|
||||
}else{
|
||||
ret << s.substring(s.indexOf(":")+1, s.size())
|
||||
}
|
||||
}
|
||||
def month = date.substring(0, 7)
|
||||
ret << month
|
||||
rets << ret
|
||||
|
||||
println ret.toString()
|
||||
|
||||
def count = map.get(ret[1])
|
||||
if(!count){
|
||||
map.put(ret[1], 1)
|
||||
}else{
|
||||
map.put(ret[1], count+1)
|
||||
}
|
||||
// FileUtils.batchCreateFile([ret], "D:\\数据\\浙商\\浙商失败案例.txt",null, "utf-8", true);
|
||||
}
|
||||
|
||||
map.each{m,r->
|
||||
println m+" "+r
|
||||
}
|
||||
// ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释")
|
||||
}
|
||||
def run(){
|
||||
def start = "2021-03-01"
|
||||
def rets = []
|
||||
rets << ["月份","总调用量","成功量","失败量"]
|
||||
while (true){
|
||||
def end = DateUtils.addMonth(DateUtils.parseFromTightYYYYMMDD(start), 1)
|
||||
// def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null)
|
||||
// def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200)
|
||||
|
||||
def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null)
|
||||
def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200)
|
||||
|
||||
def failCount = total - successCount
|
||||
println start[0..6] +" "+ total +" "+successCount +" "+ failCount
|
||||
rets << [start[0..6], total, successCount, failCount]
|
||||
start = end;
|
||||
if(end == "2023-09-01"){
|
||||
break;
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址")
|
||||
}
|
||||
|
||||
|
||||
def detail(){
|
||||
def datas = CommonDatautils.findLogs2("1007569", "/property/addressEvaluate")
|
||||
ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", datas, "估值地址")
|
||||
}
|
||||
}
|
||||
+4
@@ -84,5 +84,9 @@ public interface AddressMatchProxyService {
|
||||
* @return
|
||||
*/
|
||||
MatchDistrictResult checkMatchDistrictCorrect(CheckMatchDistrictCorrectParam param);
|
||||
|
||||
|
||||
|
||||
JSONObject addressStructure(String city, String address);
|
||||
|
||||
}
|
||||
|
||||
+23
-1
@@ -271,7 +271,29 @@ public class AddressMatchProxyServiceImpl extends AbstractHttpApiService impleme
|
||||
}
|
||||
return matchResult;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public JSONObject addressStructure(String city, String address) {
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure/v2", params, "UTF-8");
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
if (json.containsKey("result")) {
|
||||
return (JSONObject) json.get("result");
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
LOGGER.error("µØÖ·±ê×¼»¯Ê§°Ü", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取高德中分数最高的信息
|
||||
* @param city
|
||||
|
||||
Reference in New Issue
Block a user