This commit is contained in:
2023-08-31 14:38:25 +08:00
40 changed files with 1868 additions and 185 deletions
@@ -25,6 +25,7 @@ public class RegionData implements PriorityOrdered {
private final static Logger LOGGER = LoggerFactory.getLogger(RegionData.class);
private final static long REGION_RELOAD_INTERVAL = 1000;
private static DBCollection REGION_CONN = null;
private static DBCollection TOWN_CONN = null;
private static Set<String> ALL_CITY = new HashSet<String>();
private static Map<String,List<String>> cityAndAliasMap = new HashMap<String,List<String>>();
@@ -32,12 +33,15 @@ public class RegionData implements PriorityOrdered {
* 城市对应级别Map
*/
private static Map<String, Integer> cityGradeMap = new HashMap<String, Integer>();
protected final static Map<String, Map<String, Map<String, Map<String, List<String>>>>> townDataMap = new HashMap<String, Map<String, Map<String, Map<String, List<String>>>>>();
private static void init() {
try {
REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
TOWN_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1).append("alias", 1));
BasicDBObject dbo;
while(cursor.hasNext()){
@@ -58,6 +62,7 @@ public class RegionData implements PriorityOrdered {
start(REGION_RELOAD_INTERVAL);
reload();
// reloadTown();
}
private static void start(long time) {
@@ -114,6 +119,66 @@ public class RegionData implements PriorityOrdered {
LOGGER.info("RegionData reload() size [" + regionDataMap.size() + "]");
}
public static void reloadTown(){
DBObject query = new BasicDBObject();
query.put("year", 2022);
DBObject fields = new BasicDBObject();
fields.put("fangjia_city", 1);
fields.put("fangjia_area", 1);
fields.put("street_name", 1);
fields.put("alias", 1);
DBCursor cursor = TOWN_CONN.find(query, fields);
while (cursor.hasNext()) {
DBObject dbo = cursor.next();
String city = (String) dbo.get("fangjia_city");
String region = (String) dbo.get("fangjia_area");
String street = (String) dbo.get("street_name");
List<String> ps = (List<String>) dbo.get("alias");
if(!ALL_CITY.contains(city)){
continue;
}
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
if(cityRegionMap == null){
cityRegionMap = new HashMap<String, Map<String, Map<String, List<String>>>>();
townDataMap.put(city, cityRegionMap);
}
Map<String, Map<String, List<String>>> regoinMap = cityRegionMap.get(region);
if(regoinMap == null){
regoinMap = new HashMap<String, Map<String, List<String>>>();
cityRegionMap.put(region, regoinMap);
}
Map<String, List<String>> townMap = regoinMap.get("street");
if(townMap == null){
townMap = new HashMap<String, List<String>>();
regoinMap.put(street, townMap);
}
if (CollectionUtils.isNotEmpty(ps) && ps.size() > 1) {
Collections.sort(ps, new Comparator<String>() {
public int compare(String arg0, String arg1) {
if (StringUtils.isBlank(arg0)) {
return 1;
}
if (StringUtils.isBlank(arg1)) {
return -1;
}
return arg1.length() - arg0.length();
}
});
}
if(null != ps && !ps.contains(street)){
ps.add(street);
}
townMap.put(street, ps);
}
LOGGER.info("TownData reload() size [" + townDataMap.size() + "]");
}
static class ScheduleRunTask extends TimerTask {
private int nextScheduleHour;
@@ -174,6 +239,28 @@ public class RegionData implements PriorityOrdered {
return ret;
}
public static String formatStreet(String city, String region, String street) {
if (StringUtils.isBlank(region) || StringUtils.isBlank(city) || StringUtils.isBlank(street)) {
return null;
}
String ret = null;
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
if(MapUtils.isNotEmpty(cityRegionMap)){
Map<String, Map<String, List<String>>> streetMap = cityRegionMap.get(region);
if (MapUtils.isNotEmpty(streetMap)) {
for (Map<String, List<String>> crm : streetMap.values()) {
if (null != ret) {
break;
}
ret = format(crm, street);
}
}
}
return ret;
}
public static String parseRegionFormAddress(String city, String address){
if(StringUtils.isBlank(address)){
return null;
@@ -1463,6 +1463,13 @@ public class DataUtil {
return false;
}
public static boolean isNumber(String s) {
if (s != null && !"".equals(s.trim()))
return s.matches("^[0-9]*$");
else
return false;
}
public static double roundDouble(double d, int scale) {
try {
double ret = new BigDecimal(String.valueOf(d)).setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
@@ -37,6 +37,8 @@ import com.fangjia.dal.mongo.open.model.EvaluateRatio;
import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.open.dto.*;
import com.fangjia.open.dto.addressGovernance.StructureDto;
import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.service.district.*;
import com.fangjia.service.mongo.fangjia.*;
import com.fangjia.service.open.*;
@@ -1085,6 +1087,21 @@ public class PropertyController {
return mav;
}
if(model.getFloor() == null){
JSONObject structureOb = addressMatchProxyService.addressStructure(district.getCity(), model.getAddress());
if(structureOb != null){
if(structureOb.containsKey("buildingNumber")){
model.setBuildingNumber(structureOb.getString("buildingNumber"));
}
if(structureOb.containsKey("roomNumber")){
String roomNumber = structureOb.getString("roomNumber");
model.setFloor(getFloor(roomNumber));
}
}
}
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
if(loudong != null && loudong.getFloors() >= model.getFloor()){
@@ -1108,6 +1125,8 @@ public class PropertyController {
} else {
model.setCalDate(DateUtil.getFirstDayOfWeek(new Date()));
}
Evaluation evaluation = evaluationService.evaluate(district, buildEvaluation(model, request),
model.getCalDate());
Double cityPriceCredit = null;
@@ -4057,6 +4076,19 @@ public class PropertyController {
return "fangjia.com";
}
private Integer getFloor(String roomNumber){
if(StringUtils.isNotBlank(roomNumber) && DataUtil.isNumber(roomNumber)){
if(roomNumber.length() == 3){
return Integer.valueOf(roomNumber.substring(0,1));
}else if(roomNumber.length() == 4){
return Integer.valueOf(roomNumber.substring(0,2));
}else{
return null;
}
}
return null;
}
private boolean checkExists(FacetQueryResult<NewHousePojo> houseBoList, NewHousePojo house) {
for (NewHousePojo bo : houseBoList) {
boolean sameName = bo.getName().equals(house.getName());
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -40,61 +40,61 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def ccbMaps = null
if(today<=10){
ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
}else{
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
}
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
}
def allDatas = []
allDatas << headers
allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle"
if(today > 10){
delPgData()
saveToPgSql(rows)
}
// def citys = CcbCommonUtils.getRunningCitys(args)
// def rows = []
//
// def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
// //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
// //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
//
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// def today = new Date().getDate()
//
// println "starting 。。。。。。。。。"
// for(def city : citys){
//
// index ++
//
//
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
//
//
// def ccbMaps = null
// if(today<=10){
// ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
// }else{
// ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
// }
//
// def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
// rets = addDeleteDist(dists, ccbMaps, rets)
// rows.addAll(rets)
// println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
//
//// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat"
// }
//
// def allDatas = []
// allDatas << headers
// allDatas.addAll(rows)
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成->导Oracle"
//
// if(today > 10){
// delPgData()
// saveToPgSql(rows)
// }
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
// saveToPgSql2(datas)
def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
delPgData()
saveToPgSql2(datas)
}
def addDeleteDist(dists, ccbMaps, rets){
@@ -217,7 +217,7 @@ class BigdataDealBatch extends BaseBatch{
def price = ccbPriceMap.get(obj._id.toString())
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){
if(mod<=20){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
@@ -392,12 +392,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202306
def month = 202307
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202307)
def rets = findPeGteThirdParty2(city, 202308)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-04-01"
def date = "2023-05-01"
def endDate = "2023-06-01"
def newDate = "2023-07-01"
def startDate = "2023-05-01"
def date = "2023-06-01"
def endDate = "2023-07-01"
def newDate = "2023-08-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -41,11 +41,10 @@ class ExportPgsqlEvaluationResult {
exportRegionPrice(curMonth)
exportCommunityBasePrice(curMonth)
// exportCommunityBasePrice1(curMonth)
exportCommunityPdescribeGj(curMonth)
exportCommCaseDetailGj(curMonth)
//
exportInOk()
//
SSHHelperTest jschCommand = new SSHHelperTest();
@@ -1,8 +1,10 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import groovy.json.JsonSlurper
@@ -32,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilders
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
class CommonDatautils {
@@ -252,6 +255,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","华润佘山九里二期")
DBObject keys = new BasicDBObject();
if(fields){
@@ -3941,5 +3945,188 @@ class CommonDatautils {
return files;
}
static def findLogs(userId, requestUri, startDate, endDate, code){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
if(startDate){
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
queryBuilder.must(date_range_cdate);
}
if(code){
queryBuilder.must(QueryBuilders.termQuery("code", code))
}
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
return rets.size()
}
static def findLogsBySql(userId, requestUri, startDate, endDate, code){
String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' "
if(startDate){
sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'"
}
if(code){
sql = sql + " and code=${code}"
}
def total = null
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
return total;
}
static def findAmapDetailBySql(city){
String sql = "select * from amap_poi_detail where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
// if(data.name.contains('华山小区G栋')){
def point = data.location
// def tag = null
// if(data.get("business.keytag")){
// tag = data.get("business.keytag")
// }
def business_area = null
if(data.get("business.business_area")){
business_area = data.get("business.business_area")
}
def location = [:]
location.lon = point.x
location.lat = point.y
rets << ["province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
// }
})
return rets;
}
//用sql有些字段取不出来
static def findDetailLogsBySql(userId, requestUri){
String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200"
sql = sql + " order by c_date desc"
def total = null
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
def ret = []
println data.toString()
ret << data.get("parameter_map.city")
// ret << data.get("parameter_map.district")
// ret << data.get("parameter_map.keyword")
// ret << data.get("parameter_map.name")
ret << data.get("parameter_map.address")
// ret << data.get("parameter_map.size")
ret << data.get("msg")
ret << data.get("code")
ret << data.get("c_date").toString()[0..10]
rets << ret
})
return rets;
}
static def findNewAmapDetail(city) {
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city))
def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder);
def results = []
if(rets) {
for (int k = 0; k < rets.size(); k++) {
def data = rets[k];
println data
}
}
return results;
}
static def findLogs2(userId, requestUri){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
queryBuilder.mustNot(QueryBuilders.termQuery("code", 200))
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def results = []
if(rets) {
for (int k = 0; k < rets.size(); k++) {
def data = rets[k];
println data
def ret = []
println data.toString()
ret << data.parameter_map.city
ret << data.parameter_map.district
ret << data.parameter_map.name
ret << data.parameter_map.address
ret << data.msg
ret << data.code
ret << data.c_date.toString()[0..10]
results << ret
}
}
return results
}
static def httpSaveData(def newUrl,def params){
def header = [:]
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
doPost(newUrl,header, params, true, "utf-8")
}
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setUseCaches(false);
conn.setConnectTimeout(5000);
conn.setReadTimeout(5000);
conn.setRequestProperty("User-Agent", UserAgents.get());
String param = null;
BufferedWriter mWriter = null;
InputStreamReader inStream = null;
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
conn.setRequestProperty(head, headers.get(head));
}
} else {
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
}
if (isJson) {
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
param = JacksonUtil.beanToJson(params)
}
conn.connect();
//println param.toString()
//输出流
if (StringUtils.isNotEmpty(param)) {
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
mWriter.write(param);
mWriter.flush();
mWriter.close();
}
if (conn.getHeaderField("Content-Encoding") != null
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
} else {
inStream = new InputStreamReader(conn.getInputStream(), charset);
}
char[] bf = new char[1024];
int len;
while ((len = inStream.read(bf)) != -1) {
buffer.append(bf, 0, len);
}
} catch (Exception e) {
e.printStackTrace()
}
return buffer.toString();
}
}
@@ -38,10 +38,10 @@ class AmapAddressUpdate {
static main(args) {
def task = new AmapAddressUpdate()
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
def addr = task.formatAddress("福州", "北三环路201号")
println addr
// task.run(path)
def path = "d:\\数据\\小区\\福建省小区地址与高德地图V2.xlsx"
// def addr = task.formatAddress("福州", "北三环路201号")
// println addr
task.run(path)
// task.similarMatch(path)
// task.addressRun(path)
// task.addressSource(path)
@@ -52,7 +52,7 @@ class AmapAddressUpdate {
// task.updateLocation(path)//更新经纬度
}
def run(path){
def citys = ["福州"]//CommonDatautils.findAllCity()
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"]//CommonDatautils.findAllCity()
def allrets = []
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
for (def city : citys) {
@@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createNewDatas("福州")
saveToDB(ret)
def results = selectLastResultDatas(ret);
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
}
def sb = appendResult(headers, ret)
def sb = appendResult(headers, results)
DataHandleUtils.outPutFile(fileName, sb, charset);
FzFtpUtils.ftpUpload(fileName)
//生成加密数据文件并上传
FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset);
}
def selectLastResultDatas(def rets){
def rows = []
for(def r : rets){
rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]]
}
return rows;
}
def appendResult(def headers, def ret){
@@ -147,6 +157,29 @@ class DistrictPriceExportBatch extends BaseBatch{
m << 0
}
m << curDate.toInteger();
if(curPrice && dbo){
if(dbo.value_type == 1){
m << "指导价"
}else if(dbo.value_type == 2){
m << "自身样本"
}else if(dbo.value_type == 3){
m << "第三方"
}else if(dbo.value_type == 4){
m << "交易数据"
}else if(dbo.value_type == 5){
m << "周边楼盘"
}else if(dbo.value_type == 6){
m << "推导"
}else if(dbo.value_type == 9){
m << "神经网络"
}else if(dbo.value_type == 10){
m << "无方式"
}else{
m << ""
}
}else{
m << "无"
}
result << m
}
return result
@@ -300,6 +333,7 @@ class DistrictPriceExportBatch extends BaseBatch{
//按comm_id修改,小区名重新赋值,防止区域和小区名修改
udata.put("region", r[3])
udata.put("name", r[4])
udata.put("value_type", r[7])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def o = new BasicDBObject()
@@ -312,6 +346,7 @@ class DistrictPriceExportBatch extends BaseBatch{
o.put("avg_price", r[5].toInteger())
o.put("month", r[6].toInteger())
o.put("c_date", new Date())
o.put("value_type", r[7])
FZ_DISTRICT_STAT_COLL.save(o)
}
}
@@ -140,6 +140,7 @@ class DistrictSellStatBatch extends BaseBatch {
try {
totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name
def weekendPriceObj = weekendPriceMap.get(key)
if(weekendPriceObj){
def curPriceObj = curMonthlyPriceMap.get(key)
@@ -155,7 +156,7 @@ class DistrictSellStatBatch extends BaseBatch {
if(curPriceObj){
curPrice = curPriceObj.avg_price
}else{
curPrice = weekendPriceObj.avg_price
curPrice = weekendPriceObj.avg_price
}
if(weekendPriceObj.value_type!=1){
@@ -197,7 +198,9 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("commid", dist._id.toString())
data.put("mom", mom)
data.put("yoy", yoy)
data.put("avg_price", curPrice)
data.put("avg_price", weekendPriceObj.avg_price)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
}
@@ -209,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("estate_type", dist.estate_type2)
data.put("date", curMonth)
data.put("avg_price", curPrice)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
data.put("m_date", new Date())
data.put("commid", dist._id.toString())
@@ -51,59 +51,61 @@ class FzDataResultStat {
})
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
def idMaps = [:]
for(def d : dists){
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
}
def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")))
def distkeys = []
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
for(def obj : list1){
def key = obj.city + "##" + obj.region + "##" + obj.name
if(!distkeys.contains(key)){
distkeys << key
}else{
println "重复:"+key
}
map1.put(key, row)
map1.put(key, obj)
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(lastMonth)
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row)
for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name
map2.put(key, obj)
}
def rows = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
map1.each {k,list->
def row = []
def index = 0
map1.each {k,obj1->
def keys = k.split("##")
def lastObj = map2.get(k)
def lastPrice = null
if(lastObj){
lastPrice = lastObj[5]
}
def lastPrice = map2.get(k)
def markup = null
try{
if(list && lastPrice && lastPrice.toInteger()>0){
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){
markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger()
}
}catch(Exception e){
e.printStackTrace()
}
def dist = CommonDatautils.findDistrictById(list[1])
def flag = "有效"
if(dist && dist.status!=0){
flag = "失效"
def flag = "无效"
if(obj1){
flag = idMaps.get(obj1.comm_id)
}
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag]
rows << row
index++
println row
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
@@ -255,4 +257,10 @@ class FzDataResultStat {
}
return files;
}
def findFzDistrictStat(lastMonth){
def q = new BasicDBObject()
q.put("month", lastMonth)
return FZ_DISTRICT_STAT_COLL.find(q).toList()
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -76,7 +76,7 @@ class ThirdPriceExportBatch extends BaseBatch{
row << dist.name
row << o.s_date
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
def zgf_price = getLastThirdPrice(o.xingfuli_esf_price, curPrice)
row << zgf_price
if(zgf_price){
add = true;
@@ -217,8 +217,8 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.s_date){
retMap.s_date = curDate
}
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
if(!retMap.xingfuli_esf_price && d.xingfuli_esf_price){
retMap.xingfuli_esf_price = d.xingfuli_esf_price
}
if(!retMap.anjuke_esf_price && d.get("anjuke_esf_price")){
retMap.anjuke_esf_price = d.get("anjuke_esf_price")
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-06-01"
static def statDate = "2023-07-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -90,4 +90,53 @@ class FileUtils {
}
return ret;
}
static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
if(headers){
sb.append(headers.join(splitChar))
sb.append("\n");
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), splitChar))
sb.append("\n");
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -1,19 +1,19 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.city
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class CityStatExport {
@@ -28,8 +28,33 @@ class CityStatExport {
def task = new CityStatExport();
// task.run()
// task.run1()
task.regionStat()
// task.regionStat()
// task.stat()
// task.houseStat()
task.addFieldStat()
}
def addFieldStat(){
ExcelUtils.multiThreadingModify("D:\\数据\\楼盘量.xlsx", { d ->
def cityInfoDto = CommonDatautils.findCityInfo(d['B'])
if(cityInfoDto){
def detailMap = cityInfoDto.district_detail
d['D'] = detailMap.address.toDouble()*100
d['E'] = 100
d['F'] = cityInfoDto.price_coverage.toDouble()*100
}
})
}
def houseStat(){
def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
// def houseCount = findHouseCount(d['F'], 20230201)
// d['H'] = houseCount
d['J'] = findHouse(d['F'])
})
}
def stat(){
@@ -47,7 +72,7 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity();
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"]
def ret = []
ret << ["城市","小区总量"]
// ret << ["城市","楼栋总量", "房号总量"]
@@ -78,7 +103,7 @@ class CityStatExport {
ret << m
println m
}
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3")
}
@@ -124,7 +149,7 @@ class CityStatExport {
def findHouseCount(def city, def startDate){
def total = 0
try{
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
@@ -136,6 +161,18 @@ class CityStatExport {
return total
}
def findHouse(city) {
int startDate = 20230201
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
.must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses.size()
}
def findLoudongCount(String city) {
return LOUDONG.count(new BasicDBObject('city', city))
}
@@ -172,4 +172,32 @@ class FangjiaClientUtils {
}
}
public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1"
if(floor){
url = url + "&floor="+floor.toString().toDouble().toInteger()
}
if(maxFloor){
url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger()
}
if(room){
url = url + "&room="+room.toString().toDouble().toInteger()
}
if(hall){
url = url + "&hall="+hall.toString().toDouble().toInteger()
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{
// strandField(offices)
// }
// addBlock()
// addFieldValue()
// updateErrorField()
export()
test()
// export()
// test()
// addOffice()
fileAddBlock()
}
def export(){
@@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{
})
}
def fileAddBlock(){
def rets = findAllBlockRange()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d->
def location = d['T']
if(location && !d['F']){
def arrs = location.split(",")
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
if(baidu){
def lng2 = baidu.x
def lat2 = baidu.y
def lastObj = null
for (def r : rets) {
def flag = InsidePolygon.isInArea(lng2, lat2, r.points)
if (flag && r.name) {
lastObj = r;
break;
}
}
if (lastObj) {
d['F'] = lastObj.name
}
}
}
})
}
}
@@ -0,0 +1,72 @@
package com.fangjia.script.gj.es
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
def datas = findHouseCount("北京", 20230201, 20230430)
ExcelUtils.write("D:\\数据\\天弘基金.xlsx", datas, "样例")
def findHouseCount(def city, def startDate, def endDate){
def rets = []
try{
// String sql = "select * from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='ke.com' order by p_date desc"
def startMdate = DateUtils.parseFromTightByPattern(startDate.toString(),"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
rets << ["城市","区域","板块","名称","面积","单价","总价","室","厅","卫","装修","朝向","发布时间","物业类型","楼层状况","楼高","建筑结构","建筑年代","标题","URL"]
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
// QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(QueryBuilders.termQuery("root", "ke.com"))
.must(date_range).must(area_range)//.must(date_range_mdate)
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses) {
for (int k = 0; k < houses.size(); k++) {
println k
def data = houses[k];
def r = []
r << data.city
r << data.region
r << data.block
r << data.district_name
r << data.area
r << data.avg_price
r << data.total
r << data.room
r << data.hall
r << data.toilet
r << data.fitment
r << data.direction
r << data.p_date
r << data.estate_type
r << data.floor_info
r << data.height
r << data.build_structure
r << data.build_date
r << data.name
r << data.url
rets << r
if(rets.size()>500){
break;
}
}
}
}catch(Exception e){
e.printStackTrace()
}
return rets
}
@@ -25,8 +25,8 @@ class DistrictLocationBatch {
static main(args) {
def task = new DistrictLocationBatch()
// task.run()
task.deleteDist()
task.run()
// task.deleteDist()
}
def deleteDist(){
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
//广州,佛山 重庆
static main(args) {
@@ -50,7 +50,7 @@ public class BeiKeCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver = new ChromeDriver()
def citys = getAllCityUrls(driver)
driver = getLogin(driver, "https://sh.ke.com/")
@@ -93,19 +93,28 @@ public class BeiKeCrawlByChrome {
}
def checkDocument(driver, url){
driver.get(url)
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
try{
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
println "异常Url:"+url
}
}
return Jsoup.parse(s);
return driver;
}
def crawlData(driver, citys){
@@ -130,7 +139,8 @@ public class BeiKeCrawlByChrome {
def newUrl = url+"/chengjiao/"
driver.get(newUrl)
Thread.sleep(10000)
def doc = checkDocument(driver, newUrl);
driver = checkDocument(driver, newUrl);
def doc = Jsoup.parse(driver.getPageSource())
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -142,8 +152,9 @@ public class BeiKeCrawlByChrome {
regionText = "昆山"
}
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println url +" 详情链接:"+detailUrls.size()
def map = getPageDeail(driver, newCity, href, regionText)
driver = map.driver
println url +" 详情链接:"+map.detailUrls.size()
}
}
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
@@ -191,9 +202,9 @@ public class BeiKeCrawlByChrome {
def detailUrls = []
while(!stop){
def url = cityUrl + "pg${page}/"
driver.get(url)
// def doc = Jsoup.parse(driver.getPageSource());
def doc = checkDocument(driver, url);
driver = checkDocument(driver, url);
def doc = Jsoup.parse(driver.getPageSource())
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
@@ -203,10 +214,11 @@ public class BeiKeCrawlByChrome {
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
println "开始查库 "+new Date().toString()
def dbo = findTrade(durl , "贝壳找房")
println "结束查库"+new Date().toString()
if (!dbo) {
parse(city, e, region, driver, null)
driver = parse(city, e, region, driver, null)
}
// else if(dbo && !dbo.lift){
// parse(city, e, region, driver, dbo._id)
@@ -225,7 +237,10 @@ public class BeiKeCrawlByChrome {
stop = true;
}
}
return detailUrls
def m = [:]
m.driver = driver
m.detailUrls = detailUrls
return m
}
def parse(city,e, region, driver, id){
@@ -233,8 +248,10 @@ public class BeiKeCrawlByChrome {
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
driver = checkDocument(driver, detailUrl);
def detailDoc = Jsoup.parse(driver.getPageSource())
println "打开获取页面 "+new Date().toString()
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -335,18 +352,21 @@ public class BeiKeCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println "解析页面完成 "+new Date().toString()
println detail
if(id){
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
}else{
DEAL_PRICE_COLL.save(detail)
}
println "保存修改完成 "+new Date().toString()
}
}
}catch(Exception ex){
ex.printStackTrace()
println e.select(".title a").attr("href")
}
return driver
}
@@ -478,7 +498,7 @@ public class BeiKeCrawlByChrome {
}
def getLogin(driver, url){
// WebDriver driver = new ChromeDriver()
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
@@ -494,17 +514,25 @@ public class BeiKeCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("17148854005");
elemPassword.sendKeys("12qwaszx");
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
// elemUsername.sendKeys("16283166371");
// elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(10000)
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
return driver
}
@@ -34,6 +34,7 @@ public class LianjiaCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -64,7 +65,7 @@ public class LianjiaCrawlByChrome {
}
def run1(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
@@ -122,7 +123,7 @@ public class LianjiaCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
// cityMap = cityMap.reverse()
driver = getLogin(driver, "https://sh.lianjia.com/")
@@ -206,17 +207,11 @@ public class LianjiaCrawlByChrome {
}
def cityUrl = ret[1]
def newUrl = cityUrl+"chengjiao/"
try{
driver.get(newUrl)
}catch(Exception e){
println newUrl +" 链接有问题!"
continue
}
// driver.get(newUrl)
//def detailUrls = getPageUrls(driver, cityUrl)
Thread.sleep(10000)
def doc = checkDocument(driver, newUrl);
// Thread.sleep(10000)
driver = checkDocument(driver, newUrl);
def doc = Jsoup.parse(driver.getPageSource())
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
if(regions){
for(def r : regions){
@@ -226,8 +221,9 @@ public class LianjiaCrawlByChrome {
regionText = "昆山"
}
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, newCity, href, regionText)
println cityUrl +" 详情链接:"+detailUrls.size()
def map = getPageDeail(driver, newCity, href, regionText)
driver = map.driver
println newUrl +" 详情链接:"+map.detailUrls.size()
}
}
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
@@ -267,30 +263,41 @@ public class LianjiaCrawlByChrome {
}
def checkDocument(driver, url){
driver.get(url)
def s = driver.getPageSource()
def flag = true
while(flag){
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
try{
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
driver = getLogin(driver, url)
driver.get(url)
Thread.sleep(5000)
s = driver.getPageSource()
}else{
flag = false
}
}catch(Exception e){
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
}
return Jsoup.parse(s);
return driver;
}
def getPageDeail(driver, city, cityUrl, regionText){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "pg${page}/"
driver.get(url)
// driver.get(url)
//def detailUrls = getPageUrls(driver, cityUrl)
def doc = checkDocument(driver, url);
driver = checkDocument(driver, url);
def doc = Jsoup.parse(driver.getPageSource())
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
@@ -303,7 +310,7 @@ public class LianjiaCrawlByChrome {
detailUrls << durl
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver, null)
driver = parse(city, e, regionText, driver, null)
}
// else if(dbo && !dbo.lift){
// parse(city, e, regionText, driver, dbo._id)
@@ -326,7 +333,10 @@ public class LianjiaCrawlByChrome {
stop = true;
}
}
return detailUrls
def m = [:]
m.driver = driver
m.detailUrls = detailUrls
return m
}
def parse(city,e, regionText, driver, id){
@@ -334,8 +344,11 @@ public class LianjiaCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if(price){
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
// driver.get(detailUrl)
driver = checkDocument(driver, detailUrl);
def detailDoc = Jsoup.parse(driver.getPageSource())
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -450,6 +463,7 @@ public class LianjiaCrawlByChrome {
ex.printStackTrace()
println e.select(".title a").attr("href")
}
return driver;
}
@@ -567,11 +581,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityList = []
def citys= ["福州"]
def citys= ["衢州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href]
}
}
@@ -593,16 +607,23 @@ public class LianjiaCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("17148854005");
elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
// elemUsername.sendKeys("17740818975");
// elemPassword.sendKeys("13qeadzc");
// elemUsername.sendKeys("16283166371");
// elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(15000)
Thread.sleep(20000)
}catch(Exception e){
Thread.sleep(10000)
driver.close()
driver = new ChromeDriver()
Thread.sleep(15000)
driver = getLogin(driver, url)
println "异常Url:"+url
}
return driver
}
@@ -0,0 +1,179 @@
package com.fangjia.script.gj.township
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class StreetProcess {
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
final DBCollection POLYLINE235 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new StreetProcess();
// println RegionData.formatRegion("鞍山","铁东区")
task.run()
}
def run(){
def region = RegionData.formatRegion("三明市", "三元区")
println region
// standard()
// matchStreet()
// synchTownship()
}
def synchTownship(){
def q = new BasicDBObject()
// q.put("city_name","福州市")
q.put("shape",new BasicDBObject('$exists', true))
def dbcursor = TOWNSHIP.find(q)
for(def d : dbcursor){
def udata = new BasicDBObject()
udata.put("shape", d.shape)
TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
def matchStreet(){
def polylineMap = findStreetPolyline()
def q = new BasicDBObject()
// q.put("fangjia_city", "福州")
// q.put("shape",new BasicDBObject('$exists', false))
def dbcursor = TOWNSHIP.find(q)
def index = 0
for(def d : dbcursor){
index++
def streetMap = polylineMap.get(d.fangjia_city+"##"+d.fangjia_area)
if(streetMap){
def alias = d.alias
if(!alias){
alias = []
}
if(!alias.contains(d.street_name)){
alias << d.street_name
}
for(def streetName : alias){
def polyline = streetMap.get(streetName)
if(polyline){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
def udata = new BasicDBObject()
udata.put("shape", resultMap)
TOWNSHIP.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
println index + " "+ d.get("_id")
break;
}
}
}
}
}
def findStreetPolyline(){
def q = new BasicDBObject()
// q.put("fjw_city","福州")
def dbcursor = POLYLINE235.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.fjw_city+"##"+d.fjw_region
def streetMap = map.get(key)
if(!streetMap){
streetMap = [:]
map.put(key, streetMap)
}
streetMap.put(d.town, d.polyline)
}
return map;
}
def standard(){
def q = new BasicDBObject()
q.put("fjw_city",new BasicDBObject('$exists', false))
def dbcursor = POLYLINE235.find(q)
def index = 0
for(def d : dbcursor){
index ++
def city = RegionData.formatCity(d.city)
def region = RegionData.formatRegion(d.city, d.region)
// def street = null
// if(city && region){
// street = RegionData.formatStreet(city, region, d.town)
// }
def udata = new BasicDBObject()
udata.put("fjw_city", city)
udata.put("fjw_region", region)
// if(street){
// udata.put("fjw_street", street)
// }
POLYLINE235.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println index
}
}
def standard1(){
def list = TOWNSHIP.find(new BasicDBObject()).toList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def city = RegionData.formatCity(d.city_name)
if(city){
def fangjia_province = ProvinceData.getProvince(city)
def region = RegionData.formatRegion(city, d.area_name)
def udata = new BasicDBObject()
udata.put("fangjia_city", city)
udata.put("fangjia_area", region)
udata.put("fangjia_province", fangjia_province)
TOWNSHIP.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println totalCount.get()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
}
}
@@ -0,0 +1,372 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 厦门银行
* 福建省,重庆市poi数据处理
*/
@BatchOwner(Watcher.GaoJuan)
class GaoDePoiDataProcess extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static def version = new Date().getTime()
public static def TAGS = ["加油站","其它能源站","加气站","洗车场","充电站","餐饮相关场所","休闲餐饮场所","购物相关场所","商场","超级市场","家电电子卖场","花鸟鱼虫市场","家居建材市场","综合市场","特色商业街","生活服务场所","邮局","事务所","人才市场","洗浴推拿场所","体育休闲服务场所","运动场馆","娱乐场所","度假疗养场所","休闲场所","医疗保健服务场所","综合医院","专科医院","诊所","急救中心","疾病预防机构","住宿服务相关","宾馆酒店","旅馆招待所","风景名胜相关","公园广场","风景名胜","产业园区","政府及社会团体相关","政府机关","外国机构","社会团体","公检法机构","交通车辆管理","工商税务机构","科教文化场所","博物馆","展览馆","会展中心","美术馆","图书馆","科技馆","天文馆","文化宫","档案馆","文艺团体","传媒机构","学校","科研机构","培训机构","驾校","机场相关","银行","银行相关","保险公司","证券公司","财务公司","公司企业","知名企业","公司","工厂","农林牧渔基地","收费站","服务区","标志性建筑物","地名地址信息","热点地名","紧急避难场所"]
public final static DBCollection COMPANY_206_COLL = MongoUtils.getCollectionByName("business", "company", "192.168.0.206",27020)
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
static main(args) {
runBatch(new GaoDePoiDataProcess(), args)
}
@Override
void process(Object args) {
// synchMapPoi();
synchCompany()
}
def synchCompany(){
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
def citys = ["重庆市"]
for(def c : citys){
def townMap = findTownRange(c)
def companyDatas = findCompanyDatas(c)
println c+ " 加载公司数据:"+companyDatas.size()
def rows = running(companyDatas, "company", townMap)
save(c, rows)
println "完成"
}
}
def findCompanyDatas(city){
def q = new BasicDBObject()
q.put("done", 1)
q.put("geo_city", city)
q.put("geo_level", new BasicDBObject('$in', ['门牌号','兴趣点']))
return COMPANY_206_COLL.find(q).toList()
}
def synchMapPoi(){
//"三明市","漳州市","厦门市","福州市"
//三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"
def citys = ["福州市","重庆市"]
for(def city : citys){
def results = CommonDatautils.findAmapDetailBySql(city)
println results.size()
def rows = running(results, "map")
save(city, rows)
}
}
def running(results, type, townMap){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(type == "map"){
rets.addAll(createData(map, townMap))
}else if(type == "company"){
rets.addAll(createCompanyData(map, townMap))
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createCompanyData(map, townMap){
def retMap = [:]
retMap.id = new ObjectId().toString()
retMap.fjw_province = map.province_name
retMap.fjw_city = map.city_name
def fjw_region = RegionData.formatRegion(map.city_name, map.geo_district)
if(fjw_region){
retMap.fjw_district = fjw_region
}
if(!retMap.fjw_district ){
retMap.fjw_district = RegionData.formatRegion(map.city_name, map.region_name)
}
retMap.province = map.geo_province
retMap.city = map.geo_city
retMap.district = map.geo_district
retMap.cat = "企业"
retMap.name = map.ent_name
// def pointList = townMap.get(retMap.fjw_district)
def locationStr = map.geo_location
if(locationStr){
def arrs = locationStr.split(",")
retMap.township = InsidePolygon.isInArea2(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), townMap)
def gd_location = [:]
gd_location.lon = arrs[0].toString().toDouble()
gd_location.lat = arrs[1].toString().toDouble()
retMap.location = gd_location
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
if(baiduMap){
def location = [:]
location.lon = baiduMap.lng
location.lat = baiduMap.lat
retMap.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(arrs[1].toString().toDouble(), arrs[0].toString().toDouble())
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = gpsLoactionMap.lng
gpslocation.lat = gpsLoactionMap.lat
retMap.gps_location = gpslocation
}
if(!retMap.township){
def gaodeMap = getTownshipInfo(locationStr)
if(gaodeMap && gaodeMap.township){
retMap.township = gaodeMap.township
}
if(!retMap.fjw_district){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(retMap.fjw_city, gaodeMap.district)
}
}
if(!fjw_region && !retMap.fjw_district){
println map.city_name+" "+map.region_name+" 无法格式化 "+map.geo_location + " "+gaodeMap
}
retMap.fjw_district = fjw_region
}
}
retMap.address = map.reg_addr
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.ent_name, " ")
retMap.version = version
retMap.status = 1
return retMap
}
def createData(map, townMap){
def fjw_province = null
if(map.city.contains("重庆")){
fjw_province = "重庆"
}else{
fjw_province = "福建"
}
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.region)
map.id = new ObjectId().toString()
map.fjw_province = fjw_province
map.fjw_city = fjw_city
if(map.cat){
def arrs = map.cat.split(";")
map.type = arrs[0]
if(arrs.size()>=2){
map.secondary_type = arrs[1]
}
if(arrs.size()>=3){
map.small_type = arrs[2]
}
}
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = baiduMap.lng
location.lat = baiduMap.lat
map.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = gpsLoactionMap.lng
gpslocation.lat = gpsLoactionMap.lat
map.gps_location = gpslocation
}
def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat)
if(gaodeMap && gaodeMap.township){
map.township = gaodeMap.township
}
if(!fjw_region){
if(gaodeMap && gaodeMap.district){
fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
}
}
if(!fjw_region){
println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap
}
map.fjw_district = fjw_region
map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
map.version = version
if(map.secondary_type && TAGS.contains(map.secondary_type)){
map.status = 0
}else{
map.status = 1
}
return map
}
def save(city,rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient("192.168.0.12")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "[${city}] 结束时间 "+new Date()
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.district = district
map.township = township
return map;
}
def findTownRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
// def regionMap = townMap.get(d.fjw_region)
// if(!regionMap){
// regionMap = [:]
// townMap.put(d.fjw_region, regionMap)
// }
// def townInfoMap = regionMap.get(d.town)
// if(townInfoMap){
// townInfoMap = [:]
// regionMap.put()
// }
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
}
@@ -0,0 +1,124 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.GeoUtils
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class NetworkDistanceBatch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
static main(args) {
runBatch(new NetworkDistanceBatch(), args)
}
@Override
void process(Object args) {
def networks = ExcelUtils.read("D:\\数据\\厦门\\网点.xlsx", 0, 1)
def citys = ["厦门","三明","宁德","泉州","福州","漳州","南平","龙岩","莆田","重庆"]
for(def c : citys){
run(c, "district", networks)
}
}
def run(city, cat, networks){
def datas = findDistrictData(city, cat, ["city","region", "name", "category", "location"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def dist = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = createData(dist, networks)
if(map){
rets << map
}
// println city +" "+cat +" "+totalCount.get() +"/"+datas.size()+" "+dist.name+" "+dist._id
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
saveDatas(rets);
println cat +" "+city +" 总量:"+datas.size()+" 在5公里内总量:"+rets.size()
}
def saveDatas(def rets){
NETWORK_COLL.insert(rets)
}
def createData(dist, networks){
def lastRest = null
def minDistance = Integer.MAX_VALUE
for(def row : networks){
def locationstr = row['D']
if(locationstr){
def arrs = locationstr.split(",")
def distance = GeoUtils.calculateDistance(dist.location.lon, dist.location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble())
if(distance <= minDistance){
minDistance = distance
lastRest = row
}
}
}
if(minDistance<=5000){
def obj = new BasicDBObject()
obj.put("_id", dist._id);
obj.put("city", dist.city);
obj.put("region", dist.region);
obj.put("name", dist.name);
obj.put("distance", (int)minDistance);
obj.put("network", lastRest['B'])
return obj
}
return null;
}
def findDistrictData(def city, cat, fields){
DBObject q = new BasicDBObject();
q.put("status", 0);
q.put("category", cat);
q.put("city", city);
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
return SEAWEED_CONN.find(q, keys).toList();
}
}
@@ -0,0 +1,35 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
@BatchOwner(Watcher.GaoJuan)
class PoiAndSeaweedExportCSV extends BaseBatch{
static main(args) {
runBatch(new PoiAndSeaweedExportCSV(), args);
}
@Override
public void process(Object args) {
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
poiExport(citys)
}
def poiExport(citys){
def fileName = "/home/goojia/xiamen/BIGDATA_POI.csv"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\厦门\\BIGDATA_POI.csv"
}
for(def c : citys){
def rets = CommonDatautils.findNewAmapDetail(c)
FileUtils.batchCreateFileBySplitChar(rets, fileName, null, "utf-8", true, "@|@");
}
}
}
@@ -0,0 +1,176 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import com.fangjia.script.common.DataHandleUtils
class SeaweedDataSynch {
static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
static main(args) {
def task = new SeaweedDataSynch()
task.run()
// println new ObjectId("622c433afb459ab08fdab460").getDate()
}
def run(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
index++
updating(c, "district")
updating(c, "office")
updating(c, "shop")
println index +" "+ c + " 完成"
}
}
def updating(city, cat){
def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"])
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = createData(dist)
//往100 上同步
updateData(map)
//往12 es上同步
// def newUrl = targetUrl +"_doc/"+map.id
// CommonDatautils.httpSaveData(newUrl, map)
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id
} catch (Exception e) {
e.printStackTrace()
println dist.city +" "+dist.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def updateData(map){
def udata = new BasicDBObject()
map.each{k,v->
if(k!="_id"){
udata.put(k,v)
}
}
SEAWEED_CONN.update(new BasicDBObject('_id', map._id), new BasicDBObject('$set', udata), true, false)
}
def createData(dist){
def map = [:]
map._id = dist._id
map.city = dist.city
map.region = dist.region
map.name = dist.name
map.address = dist.address
map.block = dist.block
map.alias= dist.alias
map.addr_alias = dist.addr_alias
map.category = dist.cat
map.estate_type = dist.estate_type2
map.status = dist.status
map.township = dist.township
map.finish_build_time = dist.create_year?dist.create_year.toString():null
if(dist.lng){
def loc = [:]
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
map.location = loc
}
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
map.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
map.gps_location = gpslocation
}
map.build_type = dist.build_type
map.households = dist.household_count
map.committee = dist.committee
if(dist.estate_charge && dist.estate_charge.toString().isNumber()){
def fee = dist.estate_charge.toString().toFloat()
if(fee < 100){
map.property_fee = DataHandleUtils.getDoubleRound(2, fee)
}
}
if(dist.car_site_count && dist.car_site_count.toString().isNumber()){
map.car_site_count = dist.car_site_count.toString().toInteger()
}
if(dist.property_life && dist.property_life.toString().isNumber()){
map.property_life = dist.property_life.toString().toInteger()
}
map.house_property = dist.house_property
if(dist.build_area && dist.build_area.toString().isNumber()){
map.build_area = dist.build_area.toString().toDouble().toInteger()
}
if(dist.all_area && dist.all_area.toString().isNumber()){
map.floor_area = dist.all_area.toString().toDouble().toInteger()
}
map.build_structure = dist.build_structure
map.developer = dist.developer
map.property_company = dist.estate_company
map.plot_ratio = dist.cubage_percent
map.landscaping_ratio = dist.virescence_percent
if(dist.loudong_count && dist.loudong_count.toString().isNumber()){
map.building_count = dist.loudong_count.toString().toInteger()
}
map.schools = dist.schools
map.main_house_type = dist.main_type
if(dist.main_type_area && dist.main_type_area.toString().isNumber()){
map.main_house_area = DataHandleUtils.getDoubleRound(2, dist.main_type_area.toString().toFloat())
}
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
return map
}
}
@@ -272,4 +272,25 @@ class InsidePolygon {
}
return flag;
}
public static def isInArea2(lng, lat, pointMap){
def flag = false;
Set<String> keys = pointMap.keySet();
def name = null
for(String key :keys){
def prints = pointMap.get(key)
for(def pointArr : prints){
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
if(flag){
def arrs = key.split("##")
name = arrs[1];
break;
}
}
if(flag){
break;
}
}
return name;
}
}
@@ -64,7 +64,7 @@ class DataStandardBatch extends BaseBatch{
def static final SHORT_DATE = Utils.getLastMonth()
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static {
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
@@ -92,7 +92,7 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias->
println "开始加载【${city}】"
// if(city == "石家庄"){
// if(city == "福州"){
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产","幸福里APP":"幸福里"]
static main(args) {
runBatch(new SynchDistrictPrice(), args);
@@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{
def run1(){
println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), 0), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
esSources.each{esSource, mgSource->
for(def city : citys){
@@ -32,7 +32,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf"
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
@@ -0,0 +1,44 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230701", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230801", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]]
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
///property/addressEvaluate /address/financialRating /v2/property/detail
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
// if (!hashSet.contains(result)) {
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
// hashSet << result
// } else {
// println("已存在")
// }
}
if (count % 7 == 0) {
hashSet.clear()
}
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信.xlsx", data, "楼盘评情")
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390"
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
/**
* 冪帠僅蛌華硊<br>
@@ -0,0 +1,83 @@
package com.fangjia.script.zsbank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
class ZheShangBank {
static main(args) {
def task = new ZheShangBank()
// task.run()
// task.detail()
task.sample()
}
def sample(){
def runedCitys = FileUtils.readTxt("D:\\数据\\浙商\\result.txt")
def rets = []
rets << ["时间","Msg", "城市", "区域", "名称","月份"]
def map = [:]
for(def line : runedCitys){
def ret = []
def date = line.substring(line.indexOf("log.")+4, line.indexOf(":"))
def msg = line.substring(line.lastIndexOf("error :")+7, line.size())
def arrs = msg.split(" ")
ret << date
for(def i=0;i<arrs.size();i++){
def s = arrs[i]
if(i == 0){
ret << s
}else{
ret << s.substring(s.indexOf(":")+1, s.size())
}
}
def month = date.substring(0, 7)
ret << month
rets << ret
println ret.toString()
def count = map.get(ret[1])
if(!count){
map.put(ret[1], 1)
}else{
map.put(ret[1], count+1)
}
// FileUtils.batchCreateFile([ret], "D:\\数据\\浙商\\浙商失败案例.txt",null, "utf-8", true);
}
map.each{m,r->
println m+" "+r
}
// ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", rets, "风险缓释")
}
def run(){
def start = "2021-03-01"
def rets = []
rets << ["月份","总调用量","成功量","失败量"]
while (true){
def end = DateUtils.addMonth(DateUtils.parseFromTightYYYYMMDD(start), 1)
// def total = CommonDatautils.findLogs("1007569", "/address/risk", start, end, null)
// def successCount = CommonDatautils.findLogs("1007569", "/address/risk", start, end, 200)
def total = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, null)
def successCount = CommonDatautils.findLogsBySql("1007569", "/property/addressEvaluate", start, end, 200)
def failCount = total - successCount
println start[0..6] +" "+ total +" "+successCount +" "+ failCount
rets << [start[0..6], total, successCount, failCount]
start = end;
if(end == "2023-09-01"){
break;
}
}
ExcelUtils.write("D:\\数据\\浙商\\浙商调用统计.xlsx", rets, "估值地址")
}
def detail(){
def datas = CommonDatautils.findLogs2("1007569", "/property/addressEvaluate")
ExcelUtils.write("D:\\数据\\浙商\\浙商失败案例.xlsx", datas, "估值地址")
}
}
@@ -84,5 +84,9 @@ public interface AddressMatchProxyService {
* @return
*/
MatchDistrictResult checkMatchDistrictCorrect(CheckMatchDistrictCorrectParam param);
JSONObject addressStructure(String city, String address);
}
@@ -271,7 +271,29 @@ public class AddressMatchProxyServiceImpl extends AbstractHttpApiService impleme
}
return matchResult;
}
@Override
public JSONObject addressStructure(String city, String address) {
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure/v2", params, "UTF-8");
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
if (json.containsKey("result")) {
return (JSONObject) json.get("result");
}
}
}catch(Exception e){
LOGGER.error("µØÖ·±ê×¼»¯Ê§°Ü", e);
}
return null;
}
/**
* 获取高德中分数最高的信息
* @param city