Merge branch 'open_gj' of http://gitlab.fangjia.com/dev0/main into open_gj

This commit is contained in:
2021-06-02 15:36:47 +08:00
149 changed files with 21579 additions and 5050 deletions
+2 -2
View File
@@ -29,7 +29,7 @@
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-core</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.0</version>
<scope>provided</scope>
</dependency>
<dependency>
@@ -154,7 +154,7 @@
<dependency>
<groupId>com.poseidon</groupId>
<artifactId>poseidon-message</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.0.1</version>
<exclusions>
<exclusion>
<groupId>org.mongodb</groupId>
+1 -1
View File
@@ -99,7 +99,7 @@
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-web</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.0</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
@@ -12,6 +12,7 @@ import com.fangjia.service.mongo.fangjia.SeawaterService;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.sothis.core.util.Pager;
import org.sothis.web.mvc.annotation.Param;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
@@ -116,6 +117,14 @@ public class SeawaterController {
return ret;
}
public Object batchDeleteAction(@Param(name="ids") String ids) throws UnsupportedEncodingException {
JsonModel ret = new JsonModel();
if (StringUtils.isNotBlank(ids)){
seawaterService.deleteByIds(ids);
}
return ret;
}
public void setSeawaterService(SeawaterService seawaterService) {
this.seawaterService = seawaterService;
}
@@ -118,9 +118,13 @@
</td>
</tr>
</table>
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
<div>
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
<input class ="btn btn-primary" id="batchDelete" type="submit" value="批量删除" style="float: right" />
</div>
<table id="dataSheet" class="table table-hover table-condensed" name="reginTable">
<tr>
<th style="width:30px;"><input type="checkbox" id="checkAll"/>全选</th>
<th style="width:60px;">类型</th>
<th style="width:50px;">区域</th>
<th style="width:130px;">小区名</th>
@@ -142,6 +146,7 @@
<#if seawaterList??>
<#list seawaterList as it>
<tr>
<td><input type="checkbox" name="houseCheck" value="${it.id}"/></td>
<td>
<#if it.category! == 'sell'>
二手房
@@ -249,7 +254,7 @@
}
});
}
changeCity('${cityDisplay!}');
$(".hrefClick").click(function(){
@@ -336,8 +341,48 @@
$("#roomSpan").hide();
}
}
// 全选
$("#checkAll").click(function(){
$('input[name="houseCheck"]').each(function() {
var _this = this
if (_this.checked == true) {
_this.checked = false
} else {
_this.checked = true
}
})
})
$("#batchDelete").click(function(){
if(confirm('确认要删除所选的数据吗?')) {
var ids = "";
$('input:checkbox[name=houseCheck]:checked').each(function() {
var _this = $(this);
ids += _this.val() + ","
});
if (ids == "") {
alert("请选择数据")
return false;
}
jQuery.ajax({
type: 'POST',
url: '../seawater/batchDelete',
data:{ids:ids},
dataType: 'json',
timeout: 100000,
success: function(model) {
alert("批量删除操作成功");
location.reload();
},
error : function() {
//alert("操作失败");
}
});
}
});
</script>
</html>
</#compress>
@@ -277,6 +277,39 @@ public class BaiDuApiUtils {
}
return null;
}
/**
*
* @param coords
* @param from
* @return
*/
public static JSONObject geoconv(String coords,int from){
Map<String,String> params = new HashMap<String, String>();
params.put("output", "json");
params.put("ak", AK);
params.put("coords", coords);
params.put("from", String.valueOf(from));
HttpRequest request = new HttpRequest();
try{
String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){
changeAK();
result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
json = JSONObject.fromObject(result);
Thread.sleep(1000 * 60);
}
if(json.getInt("status") >= 300){
changeAK();
json = geoconv(coords, from);
}
return json.getJSONArray("result").getJSONObject(0);
}catch (Exception e){
e.printStackTrace();
}
return null;
}
public static JSONObject geoconv(String coords, int from, int to){
Map<String,String> params = new HashMap<String, String>();
@@ -312,25 +312,6 @@ public class DataUtil {
return ret;
}
/**
* 加密图片url
*
* @param u
* @param s
* @param t
* @return
*/
public static String encodePicUrl(String u, String s, String t) {
if (StringUtils.isBlank(u)) {
return StringUtils.EMPTY;
}
try {
return SecurityUtil.encode("u=" + URLEncoder.encode(u, "UTF-8") + "&s=" + s + "&t=" + t);
} catch (UnsupportedEncodingException e) {
return SecurityUtil.encode("u=" + u + "&s=" + s + "&t=" + t);
}
}
/**
* 把字符串以16进制数值显示出来
*
@@ -1805,6 +1786,25 @@ public class DataUtil {
return b2.toString();
}
/**
* 加密图片url
*
* @param u
* @param s
* @param t
* @return
*/
public static String encodePicUrl(String u, String s, String t) {
if (StringUtils.isBlank(u)) {
return StringUtils.EMPTY;
}
try {
return SecurityUtil.encode("u=" + URLEncoder.encode(u, "UTF-8") + "&s=" + s + "&t=" + t);
} catch (UnsupportedEncodingException e) {
return SecurityUtil.encode("u=" + u + "&s=" + s + "&t=" + t);
}
}
public static void main(String[] args) {
System.out.println(getFirstChar("文裕花园"));
System.out.println(getFirstChar("文怡花园"));
File diff suppressed because one or more lines are too long
@@ -205,7 +205,6 @@ public abstract class ElasticDaoImpl<E extends MongoEntity> extends AbstractJpaC
@Override
public void deleteByIds(List<String> idList) {
}
@Override
@@ -14,10 +14,8 @@ public interface SeawaterDao extends com.fangjia.dal.SeawaterDao<Seawater> {
Seawater findByIdFromEs(String id,String table);
List<Seawater> findSeawatersFromEs(String category, String city, String region, String name, List<String> alias, List<String> regionAlias, Integer startDate, Integer endDate,
Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root, String sort, boolean order, int start, int limit , String table);
List<Seawater> findHouses(String city, String region, List<String> alias, Integer minPrice, Integer maxPrice, Integer limit);
}
@@ -96,6 +96,44 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
return new ArrayList<Seawater>();
}
@Override
public void deleteById(String id) {
deleteByIds(Arrays.asList(id));
}
private Map<String, String> buildHeader() {
Base64 token = new Base64();
String authenticationEncoding = token.encodeAsString(new String("" + ES_USER + ":" + ES_PASSWORD + "").getBytes());
Map<String, String> headers = new HashMap<String, String>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Basic " + authenticationEncoding + "");
return headers;
}
@Override
public void deleteByIds(List<String> idList) {
HttpRequest request = new HttpRequest();
JSONObject reqBody = new JSONObject();
JSONObject query = new JSONObject();
JSONObject bool = new JSONObject();
JSONArray must = new JSONArray();
must.add(addTerms("_id", idList));
bool.put("must", must);
query.put("bool", bool);
reqBody.put("query", query);
JSONObject script = new JSONObject();
script.put("source", "ctx._source.status=1");
reqBody.put("script", script);
try {
request.postWithBody("http://es.fangjia.com:9200/seawater/_update_by_query", reqBody, this.buildHeader(), "UTF-8");
} catch (HttpRequestException e) {
e.printStackTrace();
}
}
private JSONObject addMatch(String name, String value){
JSONObject term = new JSONObject();
JSONObject match = new JSONObject();
@@ -339,6 +377,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
if(limit >= 0){
reqBody.put("size", limit);
}
// ²»Õ¹Ê¾É¾³ý
JSONObject param = new JSONObject();
param.put("status", 0);
JSONObject temp = new JSONObject();
temp.put("term", param);
jsonList.add(temp);
query.put("must", jsonList);
q.put("bool", query);
reqBody.put("query", q);
@@ -350,7 +396,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Basic "+authenticationEncoding+"");
try {
String response = request.postWithBody("http://es.fangjia.com:9200/"+table+"/_search", reqBody, headers, "UTF-8");
String response = request.postWithBody("http://es.fangjia.com:9200/"+table+"/_search?rest_total_hits_as_int=true", reqBody, headers, "UTF-8");
JSONObject json = JSONObject.fromObject(response);
long time2 = System.currentTimeMillis();
System.out.println(time2-time1);
+23
View File
@@ -147,5 +147,28 @@
<version>1.0.0</version>
<classifier>jdk18</classifier>
</dependency>
<!-- html转pdf -->
<dependency>
<groupId>org.xhtmlrenderer</groupId>
<artifactId>core-renderer</artifactId>
<version>R8</version>
</dependency>
<!-- 绘制折线图等 -->
<dependency>
<groupId>org.jfree</groupId>
<artifactId>jfreechart</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.xmlgraphics</groupId>
<artifactId>batik-codec</artifactId>
<version>1.11</version>
</dependency>
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-core</artifactId>
<version>0.1.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
</project>
@@ -1705,6 +1705,25 @@ public class AddressController {
return mav;
}
public Object addressSimilarityAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空"));
return mav;
}
model.setAddress(model.getAddress_1());
String result1 = addressMatchProxyService.standard(convertModelToMap(model));
model.setAddress(model.getAddress_2());
String result2 = addressMatchProxyService.standard(convertModelToMap(model));
if (result1 != null && result2 != null) {
Map<String, Object> resultMap = addressService.addressSimilarity(result1, result2);
mav.setModel(BasicModel.buildOKModel().setResult(resultMap));
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
public Object parseCityAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getProvince())) {
@@ -1765,13 +1784,14 @@ public class AddressController {
dto.append("township", address.getTownship());
dto.append("street", address.getStreet());
dto.append("streetNumber", address.getStreetNumber());
dto.append("name", building.getSuggestName());
if(StringUtils.isBlank(building.getSuggestName())){
if(StringUtils.isNotBlank(address.getParsedPropertyName())){
dto.append("name", address.getParsedPropertyName());
}else if(StringUtils.isNotBlank(address.getOrgKeywords())){
dto.append("name", address.getOrgKeywords());
}
}else{
dto.append("name", building.getSuggestName());
}
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
String addr = building.getAddress();
@@ -1789,6 +1809,7 @@ public class AddressController {
}
}
dto.append("buildingNumber", address.getBuildingNumber());
dto.append("floor", address.getFloor());
dto.append("unitNumber", address.getUnitNumber());
dto.append("roomNumber", address.getRoomNumber());
dto.append("formattedAddress", address.getFormattedAddress());
@@ -131,7 +131,7 @@ import com.google.common.collect.Sets;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
//@Authority(role = Role.USER)
@Authority(role = Role.USER)
public class PropertyController {
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
@@ -4966,6 +4966,7 @@ public class PropertyController {
* @throws NumberFormatException
* @throws IllegalAccessException
*/
@Authority(role = Role.NONE)
public Object rentReportAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
@@ -5103,8 +5104,8 @@ public class PropertyController {
//周边相似小区租金
Integer room = null;
if(model.getHouseType()!=null){
room = model.getHouseType();
if(model.getRoom()!=0){
room = model.getRoom();
}else{
room = houseService.getDistrictRoomType((float)model.getSize());
}
@@ -5164,7 +5165,6 @@ public class PropertyController {
}
public Object evaluateRentAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
@@ -5255,13 +5255,6 @@ public class PropertyController {
}
public void setHouseLoudongService(HouseLoudongService houseLoudongService) {
this.houseLoudongService = houseLoudongService;
}
@@ -1,32 +1,57 @@
package com.fangjia.open.controllers.weixin;
import com.fangjia.core.util.BeanUtils;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeocodeUtils;
import com.fangjia.core.util.*;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.SdQuery;
import com.fangjia.dal.SrQuery;
import com.fangjia.dal.mongo.model.DistrictRoomSqmetreRent;
import com.fangjia.dal.mongo.model.Evaluation;
import com.fangjia.dal.mongo.model.RentDealPrice;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.open.APIConstant;
import com.fangjia.open.compatible.model.JsonModel;
import com.fangjia.open.controllers.PropertyController;
import com.fangjia.open.dto.DistrictHouseDtao;
import com.fangjia.open.interceptors.AuthenticationBean;
import com.fangjia.open.model.BaseModel;
import com.fangjia.open.model.weixin.EvaluationModel;
import com.fangjia.open.model.weixin.PropertyEvaluationModel;
import com.fangjia.open.util.LineChartsUtils;
import com.fangjia.service.api.AutoCompleteBo;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.datastore.EvaluationService;
import com.fangjia.service.datastore.HouseService;
import com.fangjia.service.district.DistrictRoomSqmetreRentService;
import com.fangjia.service.district.RentDealPriceService;
import com.fangjia.service.mongo.fangjia.FileService;
import com.fangjia.service.mongo.fangjia.SeawaterService;
import com.fangjia.service.util.DistrictScoreUtil;
import com.lowagie.text.pdf.BaseFont;
import org.apache.batik.transcoder.Transcoder;
import org.apache.batik.transcoder.TranscoderException;
import org.apache.batik.transcoder.TranscoderInput;
import org.apache.batik.transcoder.TranscoderOutput;
import org.apache.batik.transcoder.image.PNGTranscoder;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.TermQuery;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sothis.core.util.CollectionUtils;
import org.sothis.web.mvc.ActionContext;
import org.sothis.web.mvc.annotation.Param;
import org.springframework.beans.factory.annotation.Autowired;
import org.xhtmlrenderer.pdf.ITextFontResolver;
import org.xhtmlrenderer.pdf.ITextRenderer;
import javax.servlet.http.HttpServletRequest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.lang.reflect.InvocationTargetException;
import java.text.ParseException;
import java.util.*;
/**
* 建设银行租金控制类
@@ -36,7 +61,15 @@ import java.util.Map;
*/
public class CcbRentController {
private static final Logger logger = LoggerFactory.getLogger(CcbRentController.class);
private DistrictService districtService;
private SeawaterService seawaterService;
private DistrictRoomSqmetreRentService districtRoomSqmetreRentService;
private EvaluationService evaluationService;
private HouseService houseService;
private RentDealPriceService rentDealPriceService;
private FileService fileService;
public Object indexAction(BaseModel result) {
return result;
@@ -44,7 +77,9 @@ public class CcbRentController {
public Object reportAction(EvaluationModel result, @Param(name = "address") String address, @Param(name = "totalPrice") String totalPrice) {
result.setLender(address);
result.setDealPrice(Double.valueOf(totalPrice));
if (StringUtils.isNotBlank(totalPrice)) {
result.setDealPrice(Double.valueOf(totalPrice));
}
return result;
}
@@ -52,6 +87,434 @@ public class CcbRentController {
return result;
}
public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
String result = null;
HttpRequest httpRequest = new HttpRequest();
try {
Map<String, String> headers = new HashMap<String, String>();
// &__refresh_cache=1 刷新缓存
String url = "http://open.fangjia.com/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN;
Map<String, String> params = org.apache.commons.beanutils.BeanUtils.describe(model);
result = httpRequest.post(url, params, headers,"UTF-8");
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (HttpRequestException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
return result;
}
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
@Param(name = "name") String name, @Param(name = "address") String address,
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) {
com.fangjia.open.model.PropertyEvaluationModel model = new com.fangjia.open.model.PropertyEvaluationModel();
model.setCity(city);
model.setDistrict(districtStr);
model.setName(name);
model.setAddress(address);
model.setSize(size);
model.setFloor(floor);
model.setHeight(height);
if (room != null) {
model.setRoom(room);
}
if (hall != null) {
model.setHall(hall);
}
if (toilet != null) {
model.setToilet(toilet);
}
model.setDirection(direction);
model.setFitment(fitment);
if (model.getCity().endsWith("市")) {
model.setCity(model.getCity().replaceAll("市$", ""));
}
if (StringUtils.isNotBlank(request.getParameter("calDate"))) {
Date calDate = null;
try {
calDate = org.apache.commons.lang.time.DateUtils.parseDate(request.getParameter("calDate"), new String[] { "yyyy-MM-dd" });
} catch (ParseException e) {
}
model.setCalDate(calDate);
}
/* 先到CPDB库中查找小区 */
District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(),
"district");
model.setName(district.getName());
model.setRegion(district.getRegion());
model.setDistrictName(district.getName());
if (StringUtils.isBlank(model.getPropertyType())) {
if (district.getEstateType2() != null && district.getEstateType2().contains("别墅")) {
model.setPropertyType("1");
}
}
if (model.getCalDate() != null) {
if (new Date().before(model.getCalDate())) {
model.setCalDate(DateUtil.getFirstDayOfWeek(new Date()));
} else {
model.setCalDate(DateUtil.getFirstDayOfWeek(model.getCalDate()));
}
} else {
model.setCalDate(DateUtil.getFirstDayOfWeek(new Date()));
}
Evaluation evaluation = buildEvaluation(model, request);
evaluation.setSource("api_property_evaluate_rent");
evaluation = evaluationService.evaluateRent(district, evaluation, model.getCalDate());
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0) {
evaluation.setAvgPrice((int)(evaluation.getTotalPrice()/evaluation.getArea()));
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
districtPriceMap.put("city", district.getCity());
districtPriceMap.put("district", district.getRegion());
districtPriceMap.put("name", district.getName());
districtPriceMap.put("address", district.getAddress());
districtPriceMap.put("totalPrice", evaluation.getTotalPrice());
districtPriceMap.put("totalPriceUpper", CNNMFilter.convert(String.valueOf(evaluation.getTotalPrice())));
districtPriceMap.put("cal_date", DateUtil.formatByPattern(new Date(), "yyyy/MM/dd"));
districtPriceMap.put("direction", evaluation.getDirection());
districtPriceMap.put("size", evaluation.getArea());
districtPriceMap.put("floor", model.getFloor());
districtPriceMap.put("height", model.getHeight());
districtPriceMap.put("fitment", evaluation.getFitment());
Integer startPrice = (int)(evaluation.getTotalPrice() - evaluation.getTotalPrice()*0.2);
Integer endPrice = (int)(evaluation.getTotalPrice() + evaluation.getTotalPrice()*0.2);
//周边相似小区
if (null == model.getRadius()) {// 设置默认值
model.setRadius(1000);
}
String[] fields = { "city", "region", "name", "estateType2", "id","finishBuildingDate"};
List<District> allDistricts = districtService.findNearestDistrict(district, model.getRadius(), fields, Integer.MAX_VALUE);
//按条件过滤相似小区
Integer buildYear = null;
List<District> districts = null;
if(StringUtils.isNotBlank(district.getFinishBuildingDate())){
buildYear = DistrictScoreUtil.parseStringToDate(district.getFinishBuildingDate());
if(buildYear!=null){
districts = new ArrayList<District>();
int startYear = buildYear -7;
int endYear = buildYear + 7;
for(District dist : allDistricts){
if(StringUtils.isNotBlank(dist.getFinishBuildingDate())){
int simlarDistYear = DistrictScoreUtil.parseStringToDate(district.getFinishBuildingDate());
if(simlarDistYear>=startYear && simlarDistYear<=endYear && StringUtils.isNotBlank(district.getEstateType2()) && StringUtils.isNotBlank(dist.getEstateType2()) && district.getEstateType2().equals(dist.getEstateType2())){
districts.add(dist);
}
}
}
}
}
//相似房源
Double startArea = model.getSize() - model.getSize()*0.1;
Double endArea = model.getSize() + model.getSize()*0.1;
Integer startDate = Integer.parseInt(org.apache.tools.ant.util.DateUtils.format(DateUtil.addMonthToDate(new Date(), -12), "yyyyMMdd"));
List<Seawater> houses = seawaterService.findSeawatersFromEs2("rent", district.getCity(), district.getRegion(), district.getName(), null, null, startDate, null,
startArea, endArea, startPrice, endPrice, null, null, "p_date", false, 1, 5 , null,"seawater-20*/_doc");
//无评估小区房源用周边相似小区的房源
if(null == houses || houses.size()==0){
for (District dist : districts) {
houses = seawaterService.findSeawatersFromEs2("rent", dist.getCity(), dist.getRegion(), dist.getName(), null, null, startDate, null,
startArea, endArea, startPrice, endPrice, null, null, "p_date", false, 1, 5 , null,"seawater-20*/_doc");
if(houses!=null && houses.size()>0){
break;
}
}
}
if (CollectionUtils.isNotEmpty(houses)) {
for (Seawater seawater: houses) {
if (seawater.getpDate() != null) {
seawater.setRoot(DateUtil.formatByPattern(DateUtil.parseDate(String.valueOf(seawater.getpDate()), "yyyyMMdd"), "yyyy-MM-dd"));
}
}
}
districtPriceMap.put("houses", houses);
//周边相似小区租金
if(model.getRoom() != 0){
room = model.getRoom();
}else{
room = houseService.getDistrictRoomType((float)model.getSize());
}
if (null != districts) {
if (null != room) {
List<DistrictHouseDtao> dtos = new ArrayList<DistrictHouseDtao>();
Date weekend = DateUtil.getFirstDayOfWeek(new Date());
for (District dist : districts) {
List<DistrictRoomSqmetreRent> districtRoomSqmetreRents = districtRoomSqmetreRentService.findDistrictRoomSqmetreRents(dist, 1, room);
///List<DistrictStatRent> districtStatRents = districtStatRentService.findLatestDistrictStatRent(dist, 1, room);
if (null != districtRoomSqmetreRents && districtRoomSqmetreRents.size() > 0) {
DistrictRoomSqmetreRent districtRoomSqmetreRent = districtRoomSqmetreRents.get(0);
if (districtRoomSqmetreRent.getAvgPrice() > 0) {
int startSqPrice = (int)(evaluation.getAvgPrice() - evaluation.getAvgPrice()*0.1);
int endSqPrice = (int)(evaluation.getAvgPrice() + evaluation.getAvgPrice()*0.1);
// SqmetreRent sqmetreRent = sqmetreRentService.findNearestSqmetreRent(dist.getCity(), dist.getRegion(), dist.getName(), DateUtil.formatToYYYYMMDD(weekend), "district");
if(districtRoomSqmetreRent.getAvgPrice()>=startSqPrice && districtRoomSqmetreRent.getAvgPrice()<=endSqPrice){
DistrictHouseDtao dto = new DistrictHouseDtao();
dto.setCity(dist.getCity());
dto.setRegion(dist.getRegion());
dto.setDistrictName(dist.getName());
dto.setSqAvgPrice(districtRoomSqmetreRent.getAvgPrice());
dto.setRoom(room);
dtos.add(dto);
if(dtos.size()>=5){
break;
}
}
}
}
}
districtPriceMap.put("districts", dtos);
}
}
//近一年户型租金走势
Map<String, Object> trendsMap = districtRoomSqmetreRentService.getRentTrends(district.getCity(), district.getRegion(), district.getName(), room, -12);
districtPriceMap.put("trends", trendsMap);
// 生成走势图图片,一遍转pdf
//String tradesImgUrl = buildTrendImage(trendsMap, room);
String tradesImgUrl = chartImgUrl;
//成交的租房房源
List<RentDealPrice> rentTrades = rentDealPriceService.getRentDealPrices(district.getCity(), district.getRegion(), district.getName(), 5);
if (CollectionUtils.isNotEmpty(rentTrades)) {
for (RentDealPrice price : rentTrades) {
// 返回成交日期到前端,直接取日期转pdf报错!!!!!!!!!!!!!!!!!
if (price.getTradeDate() != null) {
price.setSource(DateUtil.formatByPattern(price.getTradeDate(), "yyyy-MM-dd"));
}
}
}
districtPriceMap.put("trades", rentTrades);
districtPriceMap.put("tradesImgUrl", tradesImgUrl);
result.setResult(districtPriceMap);
}
return result;
}
private String buildTrendImage(Map<String, Object> trendsMap, Integer room) {
List<String> times = (List<String>) trendsMap.get("times");
Map<String, Object> distMap = (Map<String, Object>) ((Map<String, Object>) trendsMap.get("distMap")).get(String.valueOf(room));
if (distMap != null) {
List<Integer> values = new ArrayList<Integer>();
for (int i = 0; i < times.size(); i++) {
if (distMap.get(times.get(i)) != null) {
values.add((Integer) distMap.get(times.get(i)));
} else {
values.add(null);
}
}
// 保存到本地测试
//String url = LineChartsUtils.createChartAndSave2Png("d://test.png",room + "居室", times, values);
String url = LineChartsUtils.createChartAndUpload(fileService,room + "居室", times, values);
logger.info("租金估值图片上传至服务器:{}", url);
return url;
}
return null;
}
/**
* 通过EvaluationModel组装Evaluation实体
*
* @param model
* @param request
* @return
*/
private Evaluation buildEvaluation(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request) {
Evaluation evaluation = new Evaluation();
evaluation.setCity(model.getCity());
evaluation.setRegion(model.getRegion());
evaluation.setDistrictName(model.getName());
evaluation.setDirection(DIRECTION.get(model.getDirection()));
evaluation.setFitment(EVALUATION_MODEL_FITMENT.get(FITMENT.get(model.getDecoration())));
evaluation.setAtticArea(model.getAtticSize());
evaluation.setTerraceArea(model.getOuterSize());
evaluation.setBasementArea(model.getBasementSize());
evaluation.setGardenArea(model.getGardenSize());
String worryFacility = model.getNegativeFactor();
evaluation.setCreationDate(new Date());
evaluation.setModificationDate(new Date());
evaluation.setClientNo(model.getClientNo());
if (0 != model.getAge()) {
evaluation.setAge(model.getAge());
}
evaluation.setArea(model.getSize());
if (StringUtils.isNotBlank(model.getBuildType())) {
evaluation.setBuildType(model.getBuildType());
}
if (StringUtils.isNotBlank(model.getCarport())) {
evaluation.setCarport(model.getCarport());
}
if (model.getDecorationAge() >= 1 && model.getDecorationAge() <= 5) {
evaluation.setFitmentAge(model.getDecorationAge());
}
evaluation.setCarportPrice(model.getCarportPrice());
evaluation.setCity(model.getCity());
evaluation.setDistrictName(model.getName());
if (model.getDecorationCost() != null && model.getDecorationCost() > 0) {
evaluation.setFitmentCharge(model.getDecorationCost());
}
if (0 != model.getHall()) {
evaluation.setHall(model.getHall());
}
if (null != model.getFloor()) {
evaluation.setFloor(model.getFloor());
}
if (model.getMaxFloor() != null && 0 != model.getMaxFloor()) {
evaluation.setHeight(model.getMaxFloor());
}
if (StringUtils.isNotBlank(model.getBuildingNumber())) {
evaluation.setHouseNum(model.getBuildingNumber());
}
if (StringUtils.isNotBlank(model.getHouseProperty())) {
evaluation.setHouseProperty(model.getHouseProperty());
}
if (StringUtils.isNotBlank(model.getInsidepos())) {
evaluation.setInsidepos(model.getInsidepos());
}
if (StringUtils.isNotBlank(model.getLightingVentilation())) {
evaluation.setLightingVentilation(model.getLightingVentilation());
}
if (StringUtils.isNotBlank(model.getOtherFactor())) {
evaluation.setOtherFactor(model.getOtherFactor());
}
evaluation.setRegion(model.getDistrict());
if (0 != model.getRoom()) {
evaluation.setRoom(model.getRoom());
}
if (StringUtils.isNotBlank(model.getRoomNumber())) {
evaluation.setRoomNum(model.getRoomNumber());
}
if (StringUtils.isNotBlank(model.getSideset())) {
evaluation.setSideset(model.getSideset());
}
if (0 != model.getToilet()) {
evaluation.setToilet(model.getToilet());
}
if (StringUtils.isNotBlank(model.getFacade())) {
evaluation.setFacade(model.getFacade());
}
if (StringUtils.isNotBlank(model.getTraffice())) {
evaluation.setTraffice(model.getTraffice());
}
if (StringUtils.isNotBlank(model.getShape())) {
evaluation.setShape(model.getShape());
}
if (StringUtils.isNotBlank(model.getObstacle())) {
evaluation.setObstacle(model.getObstacle());
}
if (StringUtils.isNotBlank(model.getIndustry())) {
evaluation.setIndustry(model.getIndustry());
}
if (StringUtils.isNotBlank(model.getSupportDevice())) {
evaluation.setSupportDevice(model.getSupportDevice());
}
if (StringUtils.isNotBlank(model.getLeftPosition())) {
evaluation.setLeftPosition(model.getLeftPosition());
}
if (StringUtils.isNotBlank(model.getShopType())) {
evaluation.setShopType(model.getShopType());
}
if (StringUtils.isNotBlank(model.getShopUnique())) {
evaluation.setShopUnique(model.getShopUnique());
}
if (StringUtils.isNotBlank(model.getLocation())) {
evaluation.setLocation(model.getLocation());
}
if (model.isNewDistrict()) {
evaluation.setNewDistrict(model.isNewDistrict());
}
String ip = AnalyticsUtil.getIpAddress(ActionContext.getContext().getRequest());
evaluation.setIp(ip);
AuthenticationBean authenticationBean = (AuthenticationBean) request
.getAttribute(AuthenticationBean.ATTR_AUTHENTICATION_BEAN);
if (authenticationBean != null && authenticationBean.apiUser != null) {
evaluation.setFromUserId(String.valueOf(authenticationBean.apiUser.getId()));
}
evaluation.setSource("api_property_evaluation");
return evaluation;
}
public Object saveHighChartAsImageAction(HttpServletRequest request, HttpServletResponse response) {
String svg = request.getParameter("svg");
ByteArrayOutputStream out = null;
String url = null;
try {
svg = svg.replaceAll(":rect", "rect");
Transcoder t = new PNGTranscoder();
out = new ByteArrayOutputStream();
TranscoderInput input = new TranscoderInput(new StringReader(svg));
TranscoderOutput output = new TranscoderOutput(out);
t.transcode(input, output);
byte[] data = out.toByteArray();
String id = new ObjectId().toString();
fileService.save(id,"o", data, "image/png", "o");
url = "http://img0.fangjia.com/stc/fs/" + DataUtil.encodePicUrl(id, "o", "house");
} catch (TranscoderException e) {
e.printStackTrace();
} finally {
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
JsonModel model = new JsonModel();
AutoCompleteBo bo = new AutoCompleteBo();
bo.setAddress(url);
model.setJsonModel(bo);
return model;
}
public void downloadAction(HttpServletResponse response, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String district,
@Param(name = "name") String name, @Param(name = "address") String address,
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) throws IOException {
response.setHeader("Content-disposition", "attachment; filename=report.pdf");
response.setContentType("application/pdf;charset=UTF-8");
response.setCharacterEncoding("UTF-8");
OutputStream os = null;
try {
os = response.getOutputStream();
ITextRenderer renderer = new ITextRenderer(35, 24);
ITextFontResolver fontResolver = renderer.getFontResolver();
String fontUrl = ActionContext.getContext().getServletContext().getRealPath("/") + "fonts" + File.separator + "simsun.ttc";
fontResolver.addFont(fontUrl, BaseFont.IDENTITY_H, BaseFont.NOT_EMBEDDED);
final String path = "http://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath();
String url = path + "/ccbRent/downloadPage?city=" + DataUtil.urlEncode(city) + "&district="+DataUtil.urlEncode(district)+"&name="+DataUtil.urlEncode(name);
url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment+"&chartImgUrl="+chartImgUrl;
renderer.setDocument(url);
renderer.layout();
renderer.createPDF(os, true);
renderer.finishPDF();
os.flush();
os.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (os != null) {
os.close();
}
}
}
/**
* 方法同DistrictController#suggestAction
*/
@@ -112,8 +575,67 @@ public class CcbRentController {
return ret;
}
@Autowired
private static final Map<String, String> DIRECTION = new HashMap<String, String>() {
{
put(null, "南");
put("1", "西南");
put("2", "东南");
put("3", "南");
put("4", "西北");
put("5", "东北");
put("6", "东");
put("7", "西");
put("8", "北");
put("9", "东西");
put("10", "南北");
}
};
private static final Map<String, String> FITMENT = new HashMap<String, String>() {
{
put(null, "简装");
put("1", "毛坯");
put("2", "简装");
put("3", "精装");
put("4", "豪装");
}
};
private static final Map<String, String> EVALUATION_MODEL_FITMENT = new HashMap<String, String>() {
{
put(null, "简单装修");
put("简装", "简单装修");
put("毛坯", "毛坯");
put("精装", "精装修");
put("豪装", "豪华装修");
}
};
public void setDistrictService(DistrictService districtService) {
this.districtService = districtService;
}
public void setSeawaterService(SeawaterService seawaterService) {
this.seawaterService = seawaterService;
}
public void setDistrictRoomSqmetreRentService(DistrictRoomSqmetreRentService districtRoomSqmetreRentService) {
this.districtRoomSqmetreRentService = districtRoomSqmetreRentService;
}
public void setEvaluationService(EvaluationService evaluationService) {
this.evaluationService = evaluationService;
}
public void setHouseService(HouseService houseService) {
this.houseService = houseService;
}
public void setRentDealPriceService(RentDealPriceService rentDealPriceService) {
this.rentDealPriceService = rentDealPriceService;
}
public void setFileService(FileService fileService) {
this.fileService = fileService;
}
}
@@ -19,7 +19,24 @@ public class AddesssMatchModel extends FangjiaApiModel {
private String province;
private String debugName;
private boolean useMap;
private String address_1;
private String address_2;
public String getAddress_1() {
return address_1;
}
public void setAddress_1(String address_1) {
this.address_1 = address_1;
}
public String getAddress_2() {
return address_2;
}
public void setAddress_2(String address_2) {
this.address_2 = address_2;
}
public boolean isContainsLocation() {
return containsLocation;
}
@@ -2,6 +2,8 @@ package com.fangjia.open.model.weixin;
import com.fangjia.open.model.BaseModel;
import java.util.Map;
/**
* 建行租房估值
*
@@ -69,6 +71,8 @@ public class PropertyEvaluationModel extends BaseModel {
*/
private String fitment;
private Map<String, Object> result;
@Override
public String getCityName() {
return cityName;
@@ -166,4 +170,12 @@ public class PropertyEvaluationModel extends BaseModel {
public void setFitment(String fitment) {
this.fitment = fitment;
}
public Map<String, Object> getResult() {
return result;
}
public void setResult(Map<String, Object> result) {
this.result = result;
}
}
@@ -0,0 +1,174 @@
package com.fangjia.open.util;
import com.fangjia.core.util.DataUtil;
import com.fangjia.service.mongo.fangjia.FileService;
import org.apache.batik.transcoder.Transcoder;
import org.apache.batik.transcoder.TranscoderException;
import org.apache.batik.transcoder.TranscoderInput;
import org.apache.batik.transcoder.TranscoderOutput;
import org.apache.batik.transcoder.image.JPEGTranscoder;
import org.apache.batik.transcoder.image.PNGTranscoder;
import org.bson.types.ObjectId;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartUtilities;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.axis.NumberAxis;
import org.jfree.chart.plot.CategoryPlot;
import org.jfree.chart.plot.PlotOrientation;
import org.jfree.chart.renderer.category.LineAndShapeRenderer;
import org.jfree.data.category.DefaultCategoryDataset;
import java.awt.*;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
/**
* 折线图绘图工具类
*
* @author shenjian
* @since 2021/4/9
*/
public class LineChartsUtils {
public static void main(String[] args) throws IOException {
File file = new File("d://test.png");
List<String> columns = new ArrayList<String>();
columns.add("2020-02");
columns.add("2020-03");
columns.add("2020-04");
columns.add("2020-05");
columns.add("2020-06");
columns.add("2020-07");
columns.add("2020-08");
columns.add("2020-09");
columns.add("2020-10");
columns.add("2020-11");
columns.add("2020-12");
columns.add("2021-01");
List<Integer> values = new ArrayList<Integer>();
values.add(1000);
values.add(2000);
values.add(2000);
values.add(2000);
values.add(2000);
values.add(2000);
values.add(2000);
values.add(3000);
values.add(3000);
values.add(3000);
values.add(3000);
values.add(3000);
createChartAndSave2Png("d://test.png", "1居室", columns, values);
}
public static String createChartAndSave2Png(String path, String rowKey, List<String> columns, List<Integer> values) {
File file = new File(path);
try {
ChartUtilities.saveChartAsPNG(file, createChart(rowKey, columns, values),800,300);
} catch (IOException e) {
e.printStackTrace();
}
return path;
}
public static String createChartAndUpload(FileService fileService, String rowKey, List<String> columns, List<Integer> values) {
ByteArrayOutputStream os = new ByteArrayOutputStream();
try {
ChartUtilities.writeChartAsPNG(os, createChart(rowKey, columns, values),800,300);
byte[] data = os.toByteArray();
String id = new ObjectId().toString();
fileService.save(id,"o", data, "image/png", "o");
String url = "http://img0.fangjia.com/stc/fs/" + DataUtil.encodePicUrl(id, "o", "house");
return url;
} catch (Exception e) {
e.printStackTrace();
} finally {
if (os != null) {
try {
os.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return null;
}
public static void svg2ImgTest() {
String svg = "<svg height=\"200\" xmlns=\"http://www.w3.org/2000/svg\"><polygon points=\"10,10 75,150 150,60\" style=\"fill:#63BCF7;stroke:black;stroke-width:3\"/></svg>";
String type="image/png";
String filename="svg";
if (null != type && null != svg) {
svg = svg.replaceAll(":rect", "rect");
String ext = "";
Transcoder t = null;
if (type.equals("image/png")) {
ext = "png";
t = new PNGTranscoder();
}
else if (type.equals("image/jpeg")) {
ext = "jpg";
t = new JPEGTranscoder();
}
if (null != t) {
OutputStream out = null;
try {
out = new FileOutputStream("D:\\"+filename+"."+ext);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
TranscoderInput input = new TranscoderInput(new StringReader(svg));
TranscoderOutput output = new TranscoderOutput(out);
try{
t.transcode(input, output);
System.out.println("out");
}
catch (TranscoderException e) {
e.printStackTrace();
}
}
}
}
/**
* 定义图表对象
* @param rowKey 曲线名,如一居室
* @param columns 横坐标名
* @param values 纵坐标值
*/
public static JFreeChart createChart(String rowKey, List<String> columns, List<Integer> values) {
createDataset(rowKey, columns, values);
// 折线图名称, 横坐标名称, 纵坐标名称 数据水平竖直 PlotOrientation.HORIZONTAL, 水平显示图像 true, 水平显示图像 true, include legend true, tooltips false, urls
JFreeChart chart = ChartFactory.createLineChart("", "", "", createDataset(rowKey, columns, values), PlotOrientation.VERTICAL, true, true, false);
CategoryPlot plot = chart.getCategoryPlot();
plot.setRangeGridlinesVisible(false); // 是否显示格子线
plot.setBackgroundAlpha(0.3f); // 设置背景透明度
LineAndShapeRenderer renderer = (LineAndShapeRenderer) plot.getRenderer();
renderer.setSeriesPaint(0, Color.BLUE);
NumberAxis rangeAxis = (NumberAxis) plot.getRangeAxis();
rangeAxis.setStandardTickUnits(NumberAxis.createIntegerTickUnits());
rangeAxis.setAutoRangeIncludesZero(true);
rangeAxis.setUpperMargin(0.20);
rangeAxis.setLabelAngle(Math.PI / 2.0);
return chart;
}
private static DefaultCategoryDataset createDataset(String rowKey, List<String> columns, List<Integer> values) {
DefaultCategoryDataset dataset = new DefaultCategoryDataset();
for (int i = 0; i < columns.size(); i++) {
String column = columns.get(i);
if (values.get(i) != null) {
dataset.addValue(Double.valueOf(values.get(i)), rowKey, column);
} else {
dataset.addValue(null, rowKey, column);
}
}
return dataset;
}
}
@@ -1,37 +1,45 @@
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
<#compress>
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/html">
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta name="referrer" content="no-referrer"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<title>租金报告中心</title>
<meta name="viewport" content="initial-scale=1, maximum-scale=1">
<link rel="shortcut icon" href="/favicon.ico">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">
<meta name="viewport" content="initial-scale=1, maximum-scale=1"/>
<link rel="shortcut icon" href="/favicon.ico"/>
<meta name="apple-mobile-web-app-capable" content="yes"/>
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css">
<link rel="stylesheet" href="../static/style/wx.css">
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="../static/style/wx.css"/>
<style type="text/css">
body {
overflow: hidden;
overflow-y: scroll;
background: white;
}
.main {
margin: 4px 8px 4px 8px;
background: #f8f8f8;
background: white;
}
.title {
position: relative;
color: #1E90FF;
font-size: 20px;
font-weight: bold;
}
.title1 {
font-size: 14px;
font-size: 18px;
font-weight: bold;
color: #1E90FF;
margin-top: 2px;
margin-bottom: 2px;
}
.p {
font-size: 12px;
font-size: 18px;
line-height: 30px;
}
.table {
width: 100%;
@@ -63,11 +71,9 @@
</head>
<body>
<div class="main">
<div class="title">
估值报告
<span class="download" id="download">下载</span>
估值报告<span class="download" id="download">下载</span>
</div>
<div>
<div class="title1">1 评估申明</div>
@@ -76,12 +82,12 @@
<div>
<div class="title1">2 评估时间</div>
<div class="p">
<span name="cal_date"></span><br>
评估结果<br>
楼盘名称:<span name="name"></span><br>
楼盘地址:<span name="address"></span><br>
租金估值(RMB):<span name="totalPrice"></span>元/月<br>
大写(人民币):<span name="totalPriceUpper"></span><br>
<span name="cal_date"></span><br/>
<b>评估结果</b><br/>
<b>楼盘名称</b>:<span name="name"></span><br/>
<b>楼盘地址</b>:<span name="address"></span><br/>
<b>租金估值(RMB)</b>:<span name="totalPrice"></span>元/月<br/>
<b>大写(人民币)</b>:<span name="totalPriceUpper"></span><br/>
</div>
</div>
<div>
@@ -91,20 +97,21 @@
<div>
<div class="title1">4 房产状况</div>
<div class="p">
详细地址:<span name="address"></span><br>
楼盘名称:<span name="name"></span><br>
室号:<span name="room"></span><br>
朝向:<span name="direction"></span><br>
建筑面积:<span name="size"></span>平米<br>
所在楼层:<span name="floor"></span> 层 <span name="height"></span>总层数<br>
电梯配套:<span name="hasLift"></span><br>
装修情况:<span name="fitment"></span><br>
建筑年代:<span name="buildYear"></span><br>
<b>详细地址</b>:<span name="address"></span><br/>
<b>楼盘名称</b>:<span name="name"></span><br/>
<b>室号</b>:<span name="room"></span><br/>
<b>朝向</b>:<span name="direction"></span><br/>
<b>建筑面积</b>:<span name="size"></span>平米<br/>
<b>所在楼层</b>:<span name="floor"></span> 层 <span name="height"></span>总层数<br/>
<b>电梯配套</b>:<span name="hasLift"></span><br/>
<b>装修情况</b>:<span name="fitment"></span><br/>
<b>建筑年代</b>:<span name="buildYear"></span><br/>
</div>
</div>
<div>
<div class="title1">5 历史租金</div>
<div class="charts" id="districtRentTrendsChart"></div>
<input type="hidden" id="chartImgUrl"/>
</div>
<div>
<div class="title1">6 已租房源</div>
@@ -123,29 +130,8 @@
<th>成交日期</th>
</tr>
</thead>
<tbody>
<tr class="">
<td class="td01">上海</td>
<td class="td02">浦东区</td>
<td>永泰花苑</td>
<td>***</td>
<td>82</td>
<td>2</td>
<td>0</td>
<td>普通装修</td>
<td>2020-07-29</td>
</tr>
<tr class="bg">
<td class="td01">上海</td>
<td class="td02">浦东区</td>
<td>永泰花苑</td>
<td>***</td>
<td>82</td>
<td>2</td>
<td>0</td>
<td>普通装修</td>
<td>2020-07-29</td>
</tr>
<tbody id="trades">
<td colspan="6">暂无数据</td>
</tbody>
</table>
</div>
@@ -168,7 +154,7 @@
</tr>
</thead>
<tbody id="referenceHouses">
<td colspan="6" style="text-align:center">暂无数据</td>
<td colspan="6">暂无数据</td>
</tbody>
</table>
</div>
@@ -189,7 +175,7 @@
</thead>
<tbody id="sameDistricts">
<tr class="">
<td colspan="6" style="text-align:center">暂无数据</td>
<td colspan="6">暂无数据</td>
</tr>
</tbody>
</table>
@@ -199,44 +185,87 @@
<script type='text/javascript' src='http://g.alicdn.com/sj/lib/zepto/zepto.js' charset='utf-8'></script>
<script type='text/javascript' src='http://g.alicdn.com/msui/sm/0.6.2/js/sm.js' charset='utf-8'></script>
<script type="text/javascript" src="http://cdn.hcharts.cn/highcharts/highcharts.js"></script>
<!-- 暂时注释 -->
<script type="text/javascript" src="http://cdn.highcharts.com.cn/highcharts/9.0.1/highcharts.js"></script>
<script type="text/javascript" src="https://code.highcharts.com.cn/highcharts/modules/exporting.js"></script>
<script type='text/javascript' src='../static/js/wx.js' charset='utf-8'></script>
<script type="text/javascript" src="../static/js/trend-charts.js"></script>
<script src="https://html2canvas.hertzen.com/dist/html2canvas.js"></script>
<#-- <script src="../static/js/purify.min.js"></script>-->
<#-- <script src="https://cdnjs.cloudflare.com/ajax/libs/jspdf/2.3.1/jspdf.umd.min.js"></script>-->
<#-- <script src="../static/js/jspdf.umd.js"></script>-->
<#-- <script src="../static/js/SourceHanSans-normal.js"></script>-->
<#-- <script src="../static/js/simsun-normal.js"></script>-->
<#-- <script type="text/javascript">-->
<#-- $("#download").bind('click', function(){-->
<#-- window.location.href = "download"-->
<#-- // 打印所有可用的字体-->
<#-- // jspdf.html(document.body, {-->
<#-- // callback: function (doc) {-->
<#-- // doc.save("租金报告.pdf");-->
<#-- // },-->
<#-- // x: 10,-->
<#-- // y: 10-->
<#-- // });-->
<#-- // jspdf.html("test测试", {-->
<#-- // callback: function (doc) {-->
<#-- // doc.save("租金报告.pdf");-->
<#-- // },-->
<#-- // x: 10,-->
<#-- // y: 10-->
<#-- // });-->
<#-- });-->
<#-- $(function () {-->
<#-- $('span[name="name"]').text('场中小区');-->
<#-- });-->
<#-- </script>-->
<script type="text/javascript">
$("#download").bind("click", function(){
// window.jsPDF = window.jspdf.jsPDF
// var jspdf = new jsPDF();
// jspdf.addFileToVFS('SourceHanSans-Normal.ttf',normalChinese);
// jspdf.addFont('SourceHanSans-Normal.ttf', 'SourceHanSans-Normal', 'normal');
// jspdf.setFont('SourceHanSans-Light');
//console.log(jspdf.getFontList());
var chart;
$("#download").bind('click', function(){
var chartImgUrl = $('#chartImgUrl').val();
window.location.href = "download?city="+params.city+"&district="+params.district+"&name="+params.name+"&address="+params.address+"&size="+params.size+"&floor="+params.floor
+"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment+"&chartImgUrl=" + chartImgUrl;
// jspdf.html(document.body, {
// callback: function (doc) {
// doc.save("租金报告.pdf");
// },
// x: 0,
// y: 0,
// fontFaces: [{
// family: 'SimSun',
// weight: 'bold',
// //style: style,
// src: [{
// format: "truetype",
// url: "../static/fonts/SimSun.ttf"
// }]
// }]
// });
});
// 后台传参过来用
let params = {
'city': sessionStorage.city,
'district': sessionStorage.district,
'name': sessionStorage.name,
'address': sessionStorage.address,
'size': sessionStorage.size,
'floor': sessionStorage.floor,
'height': sessionStorage.height,
'room': sessionStorage.room,
'hall': sessionStorage.hall,
'toilet': sessionStorage.toilet,
'direction': sessionStorage.direction,
'fitment': sessionStorage.fitment
};
function init() {
// 后台传参过来用
<#--let params = {-->
<#-- 'city': '${cityName}',-->
<#-- 'district': '${district}',-->
<#-- 'name': '${name}',-->
<#-- 'address': '${address}',-->
<#-- 'size': '${size}',-->
<#-- 'floor': '${floor}',-->
<#-- 'height': '${height}',-->
<#-- 'room': '${room}',-->
<#-- 'hall': '${hall}',-->
<#-- 'toilet': '${toilet}',-->
<#-- 'direction': <#if direction??>${direction}<#else>''</#if>,-->
<#-- 'fitment': <#if fitment??>${fitment}<#else>''</#if>-->
<#--};-->
let params = {
'city': sessionStorage.city,
'district': sessionStorage.district,
'name': sessionStorage.name,
'address': sessionStorage.address,
'size': sessionStorage.size,
'floor': sessionStorage.floor,
'height': sessionStorage.height,
'room': sessionStorage.room,
'hall': sessionStorage.hall,
'toilet': sessionStorage.toilet,
'direction': sessionStorage.direction,
'fitment': sessionStorage.fitment
};
$.ajax({
// 注意需要加..
url: '../property/rentReport',
@@ -260,7 +289,11 @@
// data: [3800, 3900, 4000, 4100, 4200, 4200, 4200, 4200]
// }];
if (result.trends.distMap[params.room] == undefined) {
console.error("返回历史租金房间数有误,不存在" + params.room + "居室数据")
console.log("返回历史租金,不存在" + params.room + "居室数据")
$('#districtRentTrendsChart').empty();
$('#districtRentTrendsChart').removeClass("charts");
$('#districtRentTrendsChart').addClass("p table");
$('#districtRentTrendsChart').append('<table style="width: 300px"><tr colspan="6">暂无数据<td></tr></table>');
} else {
var arr = [];
result.trends.times.forEach(function (d) {
@@ -275,7 +308,6 @@
charts.xData = result.trends.times; // 横坐标值
createChartNow('districtRentTrendsChart', charts, true);
}
// 房产状况
$('span[name="room"]').text(result.room)
$('span[name="direction"]').text(result.direction)
@@ -286,22 +318,44 @@
$('span[name="fitment"]').text(result.fitment)
$('span[name="buildYear"]').text(result.buildYear)
// 已租房源
var trades = result.trades;
if (trades && trades.length > 0) {
$('#trades').empty();
trades.forEach(function(d){
var fitment = d.fitment != undefined ? d.fitment : '';
var hall = d.hall != undefined ? d.hall : '';
$('#trades').append('<tr class="bg"> \
<td class="td01">' + d.city + '</td> \
<td class="td02">' + d.region + '</td> \
<td>' + d.districtName + '</td> \
<td>' + (d.totalPrice != undefined ? d.totalPrice : '') + '</td> \
<td>' + (d.area != undefined ? d.area : '') + '</td> \
<td>' + (d.room != undefined ? d.room : '') + '</td> \
<td>' + hall + '</td> \
<td>' + fitment + '</td> \
<td>' + formatDate(d.tradeDate) + '</td> \
</tr>');
})
}
// 样本参考
var referenceHouses = result.houses;
if (referenceHouses && referenceHouses.length > 0) {
$('#referenceHouses').empty();
referenceHouses.forEach(function(d){
var fitment = d.fitment != undefined ? d.fitment : '-';
var fitment = d.fitment != undefined ? d.fitment : '';
var date = (d.pDate + '').substring(0, 4) + "-" + (d.pDate + '').substring(4, 6) + "-" + (d.pDate + '').substring(6, 8)
$('#referenceHouses').append('<tr class="bg"> \
<td class="td01">' + d.city + '</td> \
<td class="td02">' + d.region + '</td> \
<td>' + d.districtName + '</td> \
<td>' + d.totalPrice + '</td> \
<td>' + d.area + '</td> \
<td>' + d.room + '</td> \
<td>' + d.hall + '</td> \
<td>' + (d.totalPrice != undefined ? d.totalPrice : '') + '</td> \
<td>' + (d.area != undefined ? d.area : '') + '</td> \
<td>' + (d.room != undefined ? d.room : '') + '</td> \
<td>' + (d.hall != undefined ? d.hall : '')+ '</td> \
<td>' + fitment + '</td> \
<td>' + d.pDate + '</td> \
<td>' + date + '</td> \
</tr>');
})
@@ -315,9 +369,9 @@
$('#sameDistricts').append('<tr class=""> \
<td class="td01">' + d.city + '</td> \
<td class="td02">' + d.region + '</td> \
<td>' + d.districtName + '</td> \
<td>' + d.roomPrice + '</td> \
<td>' + d.room + '</td> \
<td>' + (d.districtName != undefined ? d.districtName:'' ) + '</td> \
<td>' + (d.roomPrice != undefined ? d.roomPrice:'') + '</td> \
<td>' + (d.room != undefined ? d.room:'') + '</td> \
<td>1</td> \
</tr>');
})
@@ -331,6 +385,15 @@
}
})
};
var formatDate = function (dateStr) {
var date = new Date(dateStr);
var y = date.getFullYear();
var m = date.getMonth() + 1;
m = m < 10 ? ('0' + m) : m;
var d = date.getDate();
d = d < 10 ? ('0' + d) : d;
return y + '-' + m + '-' + d;
};
function createChartNow(renderId, data, showTradeCount) {
if(data == null || data == 'undefined' || typeof(data) == undefined) {
return;
@@ -373,8 +436,9 @@
var config = {yaxis : yaxis};
createHighChartNow(renderId, data, config);
};
function createHighChartNow(renderId, data, config) {
var chart = new Highcharts.Chart({
chart = new Highcharts.Chart({
credits:{
enabled:true,
text: '',
@@ -393,6 +457,9 @@
renderTo: renderId,
type: 'spline'
},
exporting: {
enabled: false
},
title: {
text: data.title,
floating: false,
@@ -421,12 +488,31 @@
if(this.series.name.indexOf('成交量') > -1) {
return Highcharts.dateFormat('%Y年%m月', this.x) +" 成交: <span style='color:#F50;font-weight: bold;'>"+ this.y +'</span>套';
}else {
return Highcharts.dateFormat('%Y-%m-%d', this.x) +'<br>'+ this.series.name+ ": <span style='color:#F50;font-weight: bold;'>"+ this.y +'</span>元/平米';
return Highcharts.dateFormat('%Y-%m-%d', this.x) +'<br/>'+ this.series.name+ ": <span style='color:#F50;font-weight: bold;'>"+ this.y +'</span>元/平米';
}
}
}
});
saveHighChartImg();
};
function saveHighChartImg() {
var svg = chart.getSVG();
$.ajax({
// 注意需要加..
url: "saveHighChartAsImage",
data: {
svg:svg
},
dataType: "json",
type: "post",
success: function (ret) {
if (ret.address != null) {
$('#chartImgUrl').val(ret.address)
}
}
})
}
$(function () {
init();
});
@@ -0,0 +1,206 @@
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
<#compress>
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta name="referrer" content="no-referrer"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<title>租金报告中心</title>
<meta name="viewport" content="initial-scale=1, maximum-scale=1"/>
<link rel="shortcut icon" href="/favicon.ico"/>
<meta name="apple-mobile-web-app-capable" content="yes"/>
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="../static/style/wx.css"/>
<link rel="stylesheet" href="../static/style/report.css"/>
<style type="text/css">
</style>
</head>
<body>
<div class="content">
<div class="t1">
<em>1</em>估价结果
</div>
<div class="st1"><em>1.1</em>评估申明:</div>
<div class="text1">建设银行遵循中立、客观、公正的原则,通过周密细致的测算,评估求得分析对象的房<br/>地产市场参考。</div>
<div class="st1"><em>1.2</em>评估时间:</div>
<div class="text1"><#if result.cal_date??>${result.cal_date}</#if></div>
<div class="st1"><em>1.3</em>评估结果:</div>
<div class="text2">
<b>楼盘名称</b>:<#if result.name??>${result.name}</#if><br/><br/>
<b>楼盘地址</b>:<#if result.address??>${result.address}</#if><br/><br/>
<b>租金估值(RMB)</b>:<#if result.totalPrice??>${result.totalPrice}</#if>元/月<br/><br/>
<b>大写(人民币)</b>:<#if result.totalPriceUpper??>${result.totalPriceUpper}</#if><br/><br/>
</div>
<div class="st1"><em>1.4</em>报告有效期:</div>
<div class="text1" style="margin-bottom: 30px">本报告在市场无较大波动的情况下,该评估价格在报告出具起三个月内有效。</div>
<div class="t1"><em>2</em>房产状况</div>
<div class="c1">
<table class="distTab" width="700" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td class="textTd" style="width: 15%;">楼盘名称</td>
<td class="textRight1" style="width: 35%;"><#if result.name??>${result.name}</#if></td>
<td class="textTd" style="width: 15%;">地址</td>
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.address??>${result.address}</#if></td>
</tr>
<tr class="tdbg">
<td class="textTd">室号</td>
<td class="textRight1"><#if result.room??>${result.room}</#if></td>
<td class="textTd">朝向</td>
<td class="textRight2" colspan="3"><#if result.direction??>${result.direction}</#if></td>
</tr>
<tr>
<td class="textTd" style="width: 15%;">建筑面积</td>
<td class="textRight1" style="width: 35%;"><#if result.size??>${result.size}</#if>平米</td>
<td class="textTd" style="width: 15%;">所在楼层</td>
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.floor??>${result.floor}</#if> 层 <#if result.height??>${result.height}</#if>总层数</td>
</tr>
<tr class="tdbg">
<td class="textTd">电梯配套</td>
<td class="textRight1"><#if result.hasLift??>${result.hasLift}</#if></td>
<td class="textTd">装修情况</td>
<td class="textRight2" colspan="3"><#if result.fitment??>${result.fitment}</#if></td>
</tr>
<tr class="tdbg">
<td class="textTd">建筑年代</td>
<td class="textRight1"><#if result.buildYear??>${result.buildYear}</#if></td>
</tr>
</tbody>
</table>
</div>
<div class="t1">
<em>3</em>历史租金
</div>
<div>
<#if result.tradesImgUrl??>
<#-- <img width="800px" height="300px" src="../static/images/test.png"></img>-->
<img width="600px" height="300px" src="${result.tradesImgUrl}"></img>
<#else>
<div class="p table">
<table class="inline">
<tbody><tr><td>暂无数据</td></tr></tbody>
</table>
</div>
</#if>
</div>
<div class="t1">
<em>4</em>已租房源
</div>
<div style="margin-bottom: 30px;">
<table class="distTab" width="780" border="0" cellspacing="0" cellpadding="0">
<thead>
<tr class="tdbg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>成交价</th>
<th>面积</th>
<th>室</th>
<th>厅</th>
<th>装修</th>
<th>成交日期</th>
</tr>
</thead>
<tbody>
<#if (result.trades??&&result.trades?size>0)>
<#list result.trades as bo>
<tr class="tdbg">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.area??>${bo.area}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}</#if></td>
<td><#if bo.fitment??>${bo.fitment}</#if></td>
<td><#if bo.source??>${bo.source}</#if></td>
</tr>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
<div class="t1">
<em>5</em>样本参考
</div>
<div style="margin-bottom: 30px;">
<table class="distTab" width="780" border="0" cellspacing="0" cellpadding="0">
<thead>
<tr class="tdbg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>租 金</th>
<th>面积</th>
<th>室</th>
<th>厅</th>
<th>装修</th>
<th>成交日期</th>
</tr>
</thead>
<tbody id="referenceHouses">
<#if (result.houses??&&result.houses?size>0)>
<#list result.houses as bo>
<tr class="tdbg">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.area??>${bo.area}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}</#if></td>
<td><#if bo.fitment??>${bo.fitment}</#if></td>
<td><#if bo.root??>${bo.root}</#if></td>
</tr>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
<div class="t1">
<em>6</em>周边同类小区租金
</div>
<div style="margin-bottom: 30px;">
<table class="distTab" width="500" border="0" cellspacing="0" cellpadding="0">
<thead>
<tr class="tdbg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>平米租金</th>
<th>室</th>
<th>厅</th>
</tr>
</thead>
<tbody id="sameDistricts">
<#if (result.districts??&&result.districts?size>0)>
<#list result.districts as bo>
<tr class="tdbg">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}<#else>1</#if></td>
</tr>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
</div>
</body>
</html>
</#compress>
@@ -30,8 +30,6 @@
<div class="item-content">
<div class="item-inner">
<input id="cityName" value="上海" name="city" type="hidden">
<!-- 区域 -->
<input value="" name="district" type="hidden">
<input value="2000" name="radius" type="hidden">
<div class="item-title label">城市:<font>上海</font></div>
</div>
@@ -45,6 +43,8 @@
<input type="text" placeholder="请输入小区名或地址" id="districtSearch" name="name" class="search_input" required>
<input type="hidden" name="address" />
</div>
<!-- 区域 -->
<input value="" name="district" type="hidden" required>
<div class="item-media"><span>(必填项)</span></div>
</div>
</div>
@@ -68,9 +68,9 @@
<div class="item-inner">
<div class="content-block">
<div class="item-title label di">户型:</div>
<input name="room" type="hidden" id="room">
<input name="hall" type="hidden" id="hall">
<input name="toilet" type="hidden" id="toilet">
<input name="room" type="hidden" id="room" placeholder="请选择室" value="1">
<input name="hall" type="hidden" id="hall" placeholder="请选择厅">
<input name="toilet" type="hidden" id="toilet" placeholder="请选择卫">
<p class="di">
<a href="#" class="create-actions01-now">1</a>室&nbsp;&nbsp;&nbsp;&nbsp;<a
href="#" class="create-actions02-now">0</a>厅&nbsp;&nbsp;&nbsp;&nbsp;<a
@@ -147,6 +147,7 @@
<div class="content-block">
<p><a id="evaluateBtn" class="button button-big button-fill">查看租金结果</a></p>
</div>
</div>
</form>
</div>
</div>
@@ -212,15 +213,54 @@
}
$("#evaluateBtn").on('click', function(){
var inputs = $("input")
for (var i = 0; i < inputs.length; i++) {
var input = $(inputs[i])
var fieldName = input.attr("placeholder").substring(3)
var fieldVal = $.trim(input.val())
if (input.attr("required") && fieldVal == "") {
if (input.attr("name").indexOf("district") != -1) {
$.toast("请选择输入地址时提示的小区")
} else {
$.toast(fieldName + "不能为空");
}
return
}
if (input.attr("type") == 'number' && fieldVal != "") {
if (input.attr("name").indexOf("size") != -1) {
if (!/^[0-9]+(.[0-9]+)?$/.test(fieldVal)) {
$.toast(fieldName + "非法");
return
}
} else {
if (!/^[0-9]+$/.test(fieldVal)) {
$.toast(fieldName + "必须是正整数");
return
}
}
var min = input.attr("min")
var max = input.attr("max")
if (max && min && (parseFloat(min) > parseFloat(fieldVal) || parseFloat(max) < parseFloat(fieldVal))) {
$.toast(fieldName + "的取值范围是:" + min + "到" + max);
return
}
}
}
if(parseFloat($("#houseHight").val()) < parseFloat($("#houseFloor").val())){
$.toast("楼层不能大于楼高");
return
}
storeSession()
if($("#houseHight").val() && parseFloat($("#houseHight").val())>=60){
$.toast("请输入合理的楼层数");
return
}
$("#evaluateBtn").prop("disabled", true)
$.ajax({
// 注意需要加..
url: '../property/evaluateRent',
url: '../ccbRent/evaluateRent',
data:$("#evaluateForm").serialize(),
dataType:"json",
type:"post",
@@ -228,11 +268,13 @@
if(data.code == 200){
window.location.href = "report?address=" + data.result.address + "&totalPrice=" + data.result.totalPrice
}else{
$.toast(data.msg);
$.toast("估价失败,请重新评估");
$("#evaluateBtn").prop("disabled", false)
}
},
error: function () {
//$.toast("服务器异常,请重新评估");
$("#evaluateBtn").prop("disabled", false)
}
})
})
@@ -3,21 +3,18 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
<title>租金报告中心</title>
<meta name="viewport" content="initial-scale=1, maximum-scale=1">
<link rel="shortcut icon" href="/favicon.ico">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css">
<link rel="stylesheet" href="../static/style/wx.css">
<meta name="viewport" content="initial-scale=1, maximum-scale=1"/>
<link rel="shortcut icon" href="/favicon.ico"/>
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="../static/style/wx.css"/>
</head>
<body>
<div class="page-group">
<div class="page page-current">
<div class="content">
Binary file not shown.

After

Width:  |  Height:  |  Size: 3.1 KiB

@@ -0,0 +1,141 @@
body {
overflow: hidden;
overflow-y: scroll;
background: white;
/** 特别注意,否则中文不生效 **/
font-family: SimSun;
}
.content {
width: 840px;
height: 100%;
margin: 0 auto;
background: #fff;
padding-left: 20px;
}
.content .t1 {
background: url(../images/fjt/bg.png) 0 0 no-repeat;
width: 820px;
height: 42px;
color: #704f1c;
font-size: 20px;
font-weight: bold;
text-indent: 14px;
line-height: 46px;
margin-bottom: 25px;
}
.content .t1 em {
font-family: "Arial";
color: #FFF;
margin-right: 20px;
}
em {
font-style: normal;
}
.content .st1 {
background: url(../images/fjt/bg.png) 0 -60px no-repeat;
width: 172px;
height: 40px;
color: #704f1c;
font-size: 18px;
font-weight: bold;
text-indent: 7px;
line-height: 46px;
margin-bottom: 10px;
}
.content .st1 em {
font-family: "Arial";
color: #FFF;
margin-right: 14px;
}
.content .text1 {
font-size: 18px;
line-height: 26px;
margin: 0 20px 10px 50px;
word-wrap: break-word;
}
.text1 td {
text-align: center
}
.content .text2 {
font-size: 18px;
line-height: 40px;
padding: 10px 20px 10px 50px;
background: #eee;
width: 750px;
margin-bottom: 30px;
}
.content .text3 {
font-size: 16px;
line-height: 40px;
padding: 10px 20px 10px 50px;
background: #eee;
width: 750px;
margin-bottom: 30px;
}
.content .c1 {
height: 200px;
}
.content .distTab {
font-size: 18px
}
.content .distTab .textTd {
font-weight: bold;
}
.content .distTab tr td {
height: 30px;
}
.content .distTab tr th {
text-align: left;
}
.content .distTab .textRight1 {
text-align: right;
padding-right: 80px;
}
.content .distTab .textRight2 {
text-align: right;
padding-right: 5px;
}
.title {
position: relative;
color: #1E90FF;
}
.title1 {
font-size: 19px;
color: #1E90FF;
margin-top: 3px;
margin-bottom: 3px;
}
.p {
font-size: 16px;
}
table {
border-collapse: separate;
text-indent: initial;
border-spacing: 2px;
}
tbody {
display: table-row-group;
vertical-align: middle;
border-color: inherit;
}
.inline {
width: 550px;
border-spacing: 0;
text-align: left;
}
.table .inline th {
color: #999;
font-weight: normal;
line-height: 30px;
text-align: left;
}
.download {
float: right;
color: #3d4145;
border: solid 1px #333;
order-radius: 3px;
font-size: 12px;
line-height: 1rem;
width: 43px;
margin-top: 10px;
}
+14 -22
View File
@@ -17,8 +17,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<source>1.8</source>
<target>1.8</target>
<encoding>GBK</encoding>
</configuration>
</plugin>
@@ -179,7 +179,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.4.1</version>
<version>4.5.10</version>
</dependency>
<dependency>
<groupId>org.apache.ant</groupId>
@@ -315,16 +315,6 @@
<artifactId>metadata-extractor</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.apache.http</groupId>
<artifactId>httpasyncclient</artifactId>
<version>4.0</version>
</dependency>
<dependency>
<groupId>org.apache.http</groupId>
<artifactId>httpcore-nio</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.sanselan</groupId>
<artifactId>sanselan</artifactId>
@@ -448,23 +438,18 @@
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>x-pack-transport</artifactId>
<version>6.4.2</version>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.11.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>x-pack-sql-jdbc</artifactId>
<version>6.4.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>6.4.2</version>
<version>7.11.2</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>7.4.0</version>
<version>8.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -491,6 +476,13 @@
<artifactId>aliyun-java-sdk-address-purification</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.eclipsesource.j2v8</groupId>
<artifactId>j2v8_win32_x86_64</artifactId>
<version>4.6.0</version>
</dependency>
</dependencies>
<repositories>
<repository>
@@ -13,7 +13,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -147,7 +147,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
def findHouseMaxArea(String city, String region, String name, List alias) {
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24)
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -13,7 +13,7 @@ import com.mongodb.client.MongoCollection
import com.mongodb.client.model.UpdateOptions
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -94,7 +94,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it->
names << it.name
}
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -129,7 +129,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
def findHouseCount(String city, String region, String name, List alias, int months) {
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months)
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -7,18 +7,14 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
import com.mongodb.Bytes
import com.mongodb.*
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement;
import java.sql.Statement
/**
*
* ÀúÊ·¾ù¼Û¼ÆËã
@@ -27,7 +23,7 @@ import java.sql.Statement;
*/
@BatchOwner(Watcher.JiangYaHui)
class HistoricalAverageRevaluationTask extends BaseBatch{
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat')
@@ -1,6 +1,6 @@
package com.fangjia.script.avgprice
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
@@ -9,18 +9,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
import com.mongodb.Bytes
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement;
import com.mongodb.*
/**
* 无样本小区历史遗漏数据补充
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "工商大楼");
// q.put("name","东寓新村")
DBObject keys = new BasicDBObject();
if(fields){
@@ -0,0 +1,148 @@
package com.fangjia.script.companyCrawl
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import java.util.concurrent.atomic.AtomicInteger
class AiqichaCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
public final static int AbuyunMaxCount = 10;
static main(args) {
def task = new AiqichaCrawl();
task.process()
}
void process(Object args) {
def url = "https://aiqicha.baidu.com/s/l?q=%E6%96%B0%E5%8F%8B&t=&p=3&s=10&o=0&f=%7B%7D "
for(def key : companyKeys){
def queryBody = null
def s = crawl(url, queryBody);
if(s){
print s
}
println s
}
}
def crawl(url, queryBody){
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","aiqicha.baidu.com")
headers.put("Connection","keep-alive")
headers.put("Cache-Control","max-age=0")
headers.put("sec-ch-ua",'"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"')
headers.put("sec-ch-ua-mobile","?0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Sec-Fetch-Site","same-origin")
headers.put("Sec-Fetch-Mode","navigate")
headers.put("Sec-Fetch-User","?1")
headers.put("Sec-Fetch-Dest","document")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
String s = doPost(url, queryBody, headers)
if(s && s.contains("window.pageData = ")){
def prefix = "window.pageData ="
def s1 = s.substring(s.indexOf(prefix)+prefix.size())
s1 = s1.substring(0, s1.indexOf("}};")+2)
JSONObject jsonObject = JSONObject.fromObject(s1);
print jsonObject
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -0,0 +1,715 @@
package com.fangjia.script.companyCrawl
import com.eclipsesource.v8.V8
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import jdk.nashorn.api.scripting.ScriptObjectMirror
import org.apache.commons.io.IOUtils;
import org.apache.http.Header
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet
import org.jsoup.Jsoup
import org.jsoup.select.Elements
import org.apache.http.impl.client.HttpClients
import javax.script.ScriptEngine
import javax.script.ScriptEngineManager;
import java.util.concurrent.atomic.AtomicInteger
class GsxtCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
public final static int AbuyunMaxCount = 10;
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static main(args) {
def task = new GsxtCrawl();
//task.process()
task.run()
}
def run(){
getObject()
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
get.setHeader("Upgrade-Insecure-Requests", "1");
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
HttpResponse response = client.execute(get);
def __jsluid_h = getJsluid(response)
def s = getResponseBodyAsString(response)
println (s)
def __jsl_clearance = getJsContent(s);
//Cookie: __jsluid_h=26cb8385493c2e424767eca767720c87; __jsl_clearance=1618899529.627|-1|ao39lP6IVSloPMvq5%2FOSBqSiPZ0%3D
def newCookie = __jsluid_h+";"+__jsl_clearance
get.setHeader("Cookie", newCookie)
HttpResponse response2 = client.execute(get);
def s2 = getResponseBodyAsString(response2)
def __jsl_clearance2 = getJsContent2(s2);
println(__jsl_clearance2)
// fristCrawl()
}
//该函数测试Java与js对象
static void getObject() throws Exception{
String str = "var _0xbe8c = ['w4LCg8Onw7o=', 'ZAjDvCw=', 'YcKOw6Fm', 'wpHCqsO3CQ==', 'UMKCXnw=', 'w7Aww6XCnQ==', 'BcKNwrvCng==', 'J2zCnsKi', 'wq02WMOx', 'RQXDrUw=', 'eCnCnDw=', 'Q8KowpzDjA==', 'YznDucOn', 'YcOrXxk=', 'w4t9PsKM', 'DMO4QmA=', 'w5HCklPDmQ==', 'w5rCv8Omw6Q=', 'NcOwc8K5', 'GUxjw7Q=', 'w7zDicK8w4I=', 'VCDDpsKv', 'cwLChT4=', 'wpvCl1vDqw==', 'wrvCvMOKNQ==', 'RTnDrAc=', 'GMKFw4hk', 'SxnDvC0=', 'H2NHNQ==', 'w6Aqw7LCqg==', 'wq0mW8Ok', 'L1zCv8Kp', 'B8OqwoI2', 'Jl7DuX8=', 'JcOBwpIR', 'U8OnfSo=', '6K2K5rOG6aqL6KyM', 'f8Kiwo/Dhg==', 'aMK6w6vDog==', 'JcKgw4FC', 'w61vJEg=', 'LMOrwoY0', 'GcK4w6lt', 'w4XDrsO5wpY=', 'J8OhwoM1', 'w5Mzw7J/', 'wocbcBA=', 'SibDt8Oj', 'w7xgwoXDkQ==', 'AcKyDEQ=', 'Rg7Dkw==', 'w6UFK8Kw', 'TTzDjcKK', 'w6QhDcKy', 'ZBLDols=', 'ZjPDrk4=', 'w6XDqX3Ctw==', 'w6EvJcKT', 'JHtUDQ==', 'wq8mI8KE', 'JMOhwoJ9', 'MsK1AkE=', 'w7fCmhcm', 'VsK2w5fDtQ==', 'w7EUw4PCnw==', 'dz/DgsKu', 'UzfDtcK0', 'wpd9fWE=', 'worDvcKFw4M=', 'w4PDr0zDhg==', 'w7ocSMKg', 'wp1uwoZd', 'eAjDv8Oz', 'aATCgTw=', 'w7UGw7k=', 'wqZCwotq', 'HFIBw5s=', 'WsOdw7JB', 'w5PDpXHCpA==', 'GMO8wqdD', 'w4TCsQs=', 'w6/DmcODwpg=', 'w7MuwrjDoQ==', 'dDHDhsOj', 'G308w70=', 'CsO6wog9', 'FcONwrVw', 'M3vCmMK4', 'w6/Dm8OvwpI=', 'QsOLw5ZM', 'TgLDicKy', 'QSHCuQA=', 'UsOlw5RI', 'w7EqwqDDrw==', 'w6jDusK5w4k=', 'OsKaw5rDlg==', 'wosLQsOk', 'DcKxw7Vo', 'w5VuIEo=', 'w7sXw6TCqw==', 'wrLDr8KRw7s=', 'wr9+wpJR', 'w4LDscO0wrQ=', 'FFIPw6U=', 'J8Ksw5rDlA==', 'd8KZwqjDgw==', 'PcKvDF4=', 'MHh2GA==', 'dgDDs0Y=', 'wpxSS1Y=', 'w7l5JMKh', 'w7N8JMKC', 'XsKhw7hR', 'QxDDqm4=', 'PcKGw5ZJ', 'w7HCuAIQ', 'E8Kmw53Dmg==', 'CMOrwok3', 'U8K9wpvDnA==', 'IW06w5U=', 'w7DDjcOGwpQ=', 'X8KlwpHDpA==', 'w6TDn8O1wpo=', 'ZjnDv08=', 'w7/Cqjsp', 'wqLDk8KYw58=', 'w6fDvMKXw4g=', 'FsK8IUY=', 'w4knw4Rh', 'w5zCu8Ogw5M=', 'wqJ0ckc=', 'T8Kgw69G', 'GF0hw74=', 'FMOkwqkH', 'wokmbMOM', 'djHDtcKW', 'FXkow64=', 'B8OHwowH', 'VSjCpQU=', 'w5vDs8K0w4s=', 'w7vCkUV1', 'w7dMIcK8', 'WMOjZyI=', 'LcKBw7po', 'BMK7HkY=', 'B8K0w7F4', 'YsK+w67Drw==', 'HlQKw7I=', 'wpLCrMKrNg==', 'w7IjcsKy', 'w6zDusKhw4o=', 'WhPDm8Om', 'dMKlw7RO', 'dcOHw5R3', 'Nkpww7M=', 'wrnCsMKLdg==', 'dDTDqMKb', 'S8KXZXE=', 'wq/CssOpJA==', 'w7J/KFw=', 'QMKSWEo=', 'woNxbEc=', 'wqVIwow=', 'TD/Dqg4=', 'w7TDpmzCiw==', 'w7HDjWDCgA==', 'w7IEw6vCoA==', 'Q8K+w4zDiQ==', 'PElvw7k=', 'cDXDsRo=', 'FMOLX0c=', 'Z8KHwojDoA==', 'TTTCqzo=', 'woXCrsK6Uw==', 'w4lhHnY=', 'w6Q/dsKj', 'KsKIwpLCtQ==', 'QQTDn8Oy', 'D8Omw7zDoQ==', 'E8KzGk0=']; (function(_0x44c192, _0xbe8c61) {\n" +
"\t\tvar _0x1533fb = function(_0x5b499f) {\n" +
"\t\t\twhile (--_0x5b499f) {\n" +
"\t\t\t\t_0x44c192['push'](_0x44c192['shift']());\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\t_0x1533fb(++_0xbe8c61);\n" +
"\t} (_0xbe8c, 0x185));\n" +
"\tvar _0x1533 = function(_0x44c192, _0xbe8c61) {\n" +
"\t\t_0x44c192 = _0x44c192 - 0x0;\n" +
"\t\tvar _0x1533fb = _0xbe8c[_0x44c192];\n" +
"\t\tif (_0x1533['GlBvhA'] === undefined) { (function() {\n" +
"\t\t\t\tvar _0xa131c3 = function() {\n" +
"\t\t\t\t\tvar _0x9829e5;\n" +
"\t\t\t\t\ttry {\n" +
"\t\t\t\t\t\t_0x9829e5 = Function('return\\x20(function()\\x20' + '{}.constructor(\\x22return\\x20this\\x22)(\\x20)' + ');')();\n" +
"\t\t\t\t\t} catch(_0x1ba3fa) {\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t\treturn _0x9829e5;\n" +
"\t\t\t\t};\n" +
"\t\t\t\tvar _0x5e86d6 = _0xa131c3();\n" +
"\t\t\t\tvar _0x67fb0c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';\n" +
"\t\t\t\t_0x5e86d6['atob'] || (_0x5e86d6['atob'] = function(_0x1cb044) {\n" +
"\t\t\t\t\tvar _0xa69f93 = String(_0x1cb044)['replace'](/=+\$/, '');\n" +
"\t\t\t\t\tvar _0x26bfb9 = '';\n" +
"\t\t\t\t\tfor (var _0x2f1d79 = 0x0,\n" +
"\t\t\t\t\t_0x3713bb, _0x4e514b, _0x29733b = 0x0; _0x4e514b = _0xa69f93['charAt'](_0x29733b++);~_0x4e514b && (_0x3713bb = _0x2f1d79 % 0x4 ? _0x3713bb * 0x40 + _0x4e514b: _0x4e514b, _0x2f1d79++%0x4) ? _0x26bfb9 += String['fromCharCode'](0xff & _0x3713bb >> ( - 0x2 * _0x2f1d79 & 0x6)) : 0x0) {\n" +
"\t\t\t\t\t\t_0x4e514b = _0x67fb0c['indexOf'](_0x4e514b);\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t\treturn _0x26bfb9;\n" +
"\t\t\t\t});\n" +
"\t\t\t} ());\n" +
"\t\t\tvar _0x38a7ff = function(_0x5dd34b, _0x3a0f2e) {\n" +
"\t\t\t\tvar _0x3e7a99 = [],\n" +
"\t\t\t\t_0x3e2c47 = 0x0,\n" +
"\t\t\t\t_0x383663,\n" +
"\t\t\t\t_0x14720e = '',\n" +
"\t\t\t\t_0x5864f4 = '';\n" +
"\t\t\t\t_0x5dd34b = atob(_0x5dd34b);\n" +
"\t\t\t\tfor (var _0x28a6d2 = 0x0,\n" +
"\t\t\t\t_0x2ea622 = _0x5dd34b['length']; _0x28a6d2 < _0x2ea622; _0x28a6d2++) {\n" +
"\t\t\t\t\t_0x5864f4 += '%' + ('00' + _0x5dd34b['charCodeAt'](_0x28a6d2)['toString'](0x10))['slice']( - 0x2);\n" +
"\t\t\t\t}\n" +
"\t\t\t\t_0x5dd34b = decodeURIComponent(_0x5864f4);\n" +
"\t\t\t\tvar _0x3091fb;\n" +
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3091fb;\n" +
"\t\t\t\t}\n" +
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb] + _0x3a0f2e['charCodeAt'](_0x3091fb % _0x3a0f2e['length'])) % 0x100;\n" +
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
"\t\t\t\t}\n" +
"\t\t\t\t_0x3091fb = 0x0;\n" +
"\t\t\t\t_0x3e2c47 = 0x0;\n" +
"\t\t\t\tfor (var _0x8c68bc = 0x0; _0x8c68bc < _0x5dd34b['length']; _0x8c68bc++) {\n" +
"\t\t\t\t\t_0x3091fb = (_0x3091fb + 0x1) % 0x100;\n" +
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb]) % 0x100;\n" +
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
"\t\t\t\t\t_0x14720e += String['fromCharCode'](_0x5dd34b['charCodeAt'](_0x8c68bc) ^ _0x3e7a99[(_0x3e7a99[_0x3091fb] + _0x3e7a99[_0x3e2c47]) % 0x100]);\n" +
"\t\t\t\t}\n" +
"\t\t\t\treturn _0x14720e;\n" +
"\t\t\t};\n" +
"\t\t\t_0x1533['sMWqUz'] = _0x38a7ff;\n" +
"\t\t\t_0x1533['AbOzCI'] = {};\n" +
"\t\t\t_0x1533['GlBvhA'] = !![];\n" +
"\t\t}\n" +
"\t\tvar _0x5b499f = _0x1533['AbOzCI'][_0x44c192];\n" +
"\t\tif (_0x5b499f === undefined) {\n" +
"\t\t\tif (_0x1533['uRkTAI'] === undefined) {\n" +
"\t\t\t\t_0x1533['uRkTAI'] = !![];\n" +
"\t\t\t}\n" +
"\t\t\t_0x1533fb = _0x1533['sMWqUz'](_0x1533fb, _0xbe8c61);\n" +
"\t\t\t_0x1533['AbOzCI'][_0x44c192] = _0x1533fb;\n" +
"\t\t} else {\n" +
"\t\t\t_0x1533fb = _0x5b499f;\n" +
"\t\t}\n" +
"\t\treturn _0x1533fb;\n" +
"\t};\n" +
"\tfunction hash(_0x1a5a87) {\n" +
"\t\tvar _0x23a3e0 = {};\n" +
"\t\t_0x23a3e0[_0x1533('0x93', 'Kp2R') + 'J'] = function(_0x5ebeb2, _0x4681fc) {\n" +
"\t\t\treturn _0x5ebeb2 ^ _0x4681fc;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x17', '(bZ%') + 'H'] = function(_0x5108c3, _0x49fe65) {\n" +
"\t\t\treturn _0x5108c3 & _0x49fe65;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x88', 'N#(C') + 'T'] = function(_0x3d3f10, _0x3f2a86) {\n" +
"\t\t\treturn _0x3d3f10 !== _0x3f2a86;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x60', '(cNt') + 'Z'] = _0x1533('0x70', 'K)PG') + 'J';\n" +
"\t\t_0x23a3e0[_0x1533('0x49', 'Y8u@') + 'h'] = function(_0xc3ff68, _0x129ab0) {\n" +
"\t\t\treturn _0xc3ff68 >> _0x129ab0;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x9c', '\$qIi') + 'S'] = function(_0x591dc1, _0x158d9d) {\n" +
"\t\t\treturn _0x591dc1 + _0x158d9d;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x7f', '(bZ%') + 'f'] = function(_0xd6deb, _0x557760) {\n" +
"\t\t\treturn _0xd6deb >> _0x557760;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x4e', 'S9Bn') + 'd'] = function(_0xd05226, _0x29b6e8) {\n" +
"\t\t\treturn _0xd05226 * _0x29b6e8;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x7e', 'y^BN') + 'n'] = function(_0x899f8d, _0x3dbd26) {\n" +
"\t\t\treturn _0x899f8d < _0x3dbd26;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x9a', '(U[3') + 'e'] = function(_0x39678a, _0x37d488) {\n" +
"\t\t\treturn _0x39678a * _0x37d488;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xa3', 'sahq') + 'U'] = function(_0x4cf877, _0x14e1b0) {\n" +
"\t\t\treturn _0x4cf877 << _0x14e1b0;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xa7', 'K#Hh') + 'u'] = function(_0x556336, _0x56b41c) {\n" +
"\t\t\treturn _0x556336 - _0x56b41c;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x3d', '(bZ%') + 'p'] = function(_0x372c20, _0x534fe1) {\n" +
"\t\t\treturn _0x372c20 >> _0x534fe1;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x1d', 'S9Bn') + 'g'] = function(_0x26a0f8, _0x4f1660) {\n" +
"\t\t\treturn _0x26a0f8 >>> _0x4f1660;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xae', ']*f@') + 'Y'] = function(_0x3423f4, _0x53e1a9) {\n" +
"\t\t\treturn _0x3423f4 | _0x53e1a9;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x26', 's4Dp') + 'P'] = function(_0x19382a, _0x220110) {\n" +
"\t\t\treturn _0x19382a | _0x220110;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x65', 's4Dp') + 'p'] = function(_0x7b95bc, _0x22e933) {\n" +
"\t\t\treturn _0x7b95bc & _0x22e933;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x0', '\$qIi') + 'V'] = function(_0x13269b, _0x227e3b) {\n" +
"\t\t\treturn _0x13269b ^ _0x227e3b;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x29', 'J21I') + 'R'] = function(_0x1ba947, _0x4b40e4) {\n" +
"\t\t\treturn _0x1ba947(_0x4b40e4);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x50', 's4Dp') + 'n'] = _0x1533('0xa', 'qRqa') + 'e';\n" +
"\t\t_0x23a3e0[_0x1533('0x42', 'ksVw') + 'b'] = function(_0x466fa6, _0x3ce8da) {\n" +
"\t\t\treturn _0x466fa6 ^ _0x3ce8da;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x2c', 'rqYb') + 'f'] = function(_0x1e608a, _0x241888) {\n" +
"\t\t\treturn _0x1e608a - _0x241888;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x78', 'J21I') + 'u'] = function(_0x564b78, _0x18d236, _0x8fd805) {\n" +
"\t\t\treturn _0x564b78(_0x18d236, _0x8fd805);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x69', 'zx(Q') + 'D'] = function(_0x3b7666, _0x47f238, _0x57b05e) {\n" +
"\t\t\treturn _0x3b7666(_0x47f238, _0x57b05e);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x5f', 'ZaCV') + 'm'] = function(_0x452fa7, _0x493d76, _0x1446e7, _0x4af814, _0x5e0f59) {\n" +
"\t\t\treturn _0x452fa7(_0x493d76, _0x1446e7, _0x4af814, _0x5e0f59);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x22', 'Rh@#') + 'W'] = function(_0x51a858, _0x440b1c) {\n" +
"\t\t\treturn _0x51a858(_0x440b1c);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x8', 'xPGU') + 'A'] = function(_0x5e96f2, _0x167b41) {\n" +
"\t\t\treturn _0x5e96f2 + _0x167b41;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x6d', 'rqYb') + 'X'] = function(_0x46cd37, _0x5d8684) {\n" +
"\t\t\treturn _0x46cd37 + _0x5d8684;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x3e', 'rqYb') + 'g'] = function(_0x26bc4b, _0x16bc46) {\n" +
"\t\t\treturn _0x26bc4b(_0x16bc46);\n" +
"\t\t};\n" +
"\t\tvar _0x179134 = _0x23a3e0;\n" +
"\t\tfunction _0x5be8c0(_0x296dd6, _0x778566) {\n" +
"\t\t\treturn _0x179134[_0x1533('0x56', 'ZaCV') + 'J'](_0x179134[_0x1533('0x7c', 'o&Q4') + 'H'](_0x296dd6, 0x7fffffff) + (_0x778566 & 0x7fffffff), _0x179134[_0x1533('0x52', 'K)PG') + 'H'](_0x296dd6, 0x80000000)) ^ _0x778566 & 0x80000000;\n" +
"\t\t}\n" +
"\t\tfunction _0x225a28(_0x2f57af) {\n" +
"\t\t\tvar _0x3c49d8 = {};\n" +
"\t\t\t_0x3c49d8[_0x1533('0x55', 'qRqa') + 'N'] = function(_0x15bfb8, _0x51e1eb) {\n" +
"\t\t\t\treturn _0x15bfb8 < _0x51e1eb;\n" +
"\t\t\t};\n" +
"\t\t\tvar _0x567e0f = _0x3c49d8;\n" +
"\t\t\tvar _0x5b2e48 = _0x1533('0xa4', 'rQf(') + _0x1533('0x4', 'rqYb') + _0x1533('0x95', 'Rh@#') + _0x1533('0x3', 'OhQr');\n" +
"\t\t\tvar _0x321f21 = '';\n" +
"\t\t\tfor (var _0x2a5528 = 0x7; _0x2a5528 >= 0x0; _0x2a5528--) {\n" +
"\t\t\t\tif (_0x179134[_0x1533('0xb2', 'Gfc!') + 'T'](_0x1533('0x2', 'w*Nn') + 'h', _0x179134[_0x1533('0x6c', 'OhQr') + 'Z'])) {\n" +
"\t\t\t\t\t_0x321f21 += _0x5b2e48[_0x1533('0x66', ')T5g') + 'At'](_0x179134[_0x1533('0x9f', 'zx(Q') + 'H'](_0x179134[_0x1533('0x5', 'OhQr') + 'h'](_0x2f57af, _0x2a5528 * 0x4), 0xf));\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\treturn _0x567e0f[_0x1533('0x5a', 'y*wC') + 'N'](t, 0x14) ? 0x5a827999: t < 0x28 ? 0x6ed9eba1: _0x567e0f[_0x1533('0x43', 'zx(Q') + 'N'](t, 0x3c) ? -0x70e44324: -0x359d3e2a;\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\treturn _0x321f21;\n" +
"\t\t}\n" +
"\t\tfunction _0xde630f(_0x10fb9a) {\n" +
"\t\t\tvar _0x5b5b6d = _0x179134[_0x1533('0x16', 'GvA9') + 'S'](_0x179134[_0x1533('0x87', 'N#(C') + 'f'](_0x179134[_0x1533('0x83', '[H[u') + 'S'](_0x10fb9a[_0x1533('0x58', 'OhQr') + 'th'], 0x8), 0x6), 0x1),\n" +
"\t\t\t_0x797793 = new Array(_0x179134[_0x1533('0x59', 'xPGU') + 'd'](_0x5b5b6d, 0x10));\n" +
"\t\t\tfor (var _0x109346 = 0x0; _0x179134[_0x1533('0x61', 'dgE&') + 'n'](_0x109346, _0x179134[_0x1533('0x90', 'y^BN') + 'e'](_0x5b5b6d, 0x10)); _0x109346++) {\n" +
"\t\t\t\tif (_0x179134[_0x1533('0x2a', 'NNf8') + 'T'](_0x1533('0x12', 'sahq') + 'y', _0x1533('0x67', 'y*wC') + 'G')) {\n" +
"\t\t\t\t\t_0x797793[_0x109346] = 0x0;\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\treturn !! [];\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\tfor (_0x109346 = 0x0; _0x109346 < _0x10fb9a[_0x1533('0x40', 'ad*k') + 'th']; _0x109346++) {\n" +
"\t\t\t\t_0x797793[_0x109346 >> 0x2] |= _0x179134[_0x1533('0xa9', '@gLq') + 'U'](_0x10fb9a[_0x1533('0x1c', 'GvA9') + _0x1533('0xf', 'OhQr') + 'At'](_0x109346), _0x179134[_0x1533('0xb3', 'S9Bn') + 'u'](0x18, (_0x109346 & 0x3) * 0x8));\n" +
"\t\t\t}\n" +
"\t\t\t_0x797793[_0x179134[_0x1533('0xb1', 'qRqa') + 'p'](_0x109346, 0x2)] |= 0x80 << 0x18 - (_0x109346 & 0x3) * 0x8;\n" +
"\t\t\t_0x797793[_0x5b5b6d * 0x10 - 0x1] = _0x10fb9a[_0x1533('0xb', '@mqx') + 'th'] * 0x8;\n" +
"\t\t\treturn _0x797793;\n" +
"\t\t}\n" +
"\t\tfunction _0x50e067(_0x45a1cc, _0x3bcee1) {\n" +
"\t\t\treturn _0x45a1cc << _0x3bcee1 | _0x179134[_0x1533('0x5b', 'Y8u@') + 'g'](_0x45a1cc, _0x179134[_0x1533('0x91', '@mqx') + 'u'](0x20, _0x3bcee1));\n" +
"\t\t}\n" +
"\t\tfunction _0x526d3b(_0xb72077, _0xed7f25, _0x5bc269, _0x572b62) {\n" +
"\t\t\tif (_0xb72077 < 0x14) return _0x179134[_0x1533('0x98', 'Gfc!') + 'Y'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x68', 'OhQr') + 'H'](~_0xed7f25, _0x572b62));\n" +
"\t\t\tif (_0x179134[_0x1533('0x9e', 'w*Nn') + 'n'](_0xb72077, 0x28)) return _0x179134[_0x1533('0x99', ')T5g') + 'J'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
"\t\t\tif (_0xb72077 < 0x3c) return _0x179134[_0x1533('0x13', 'CNzQ') + 'P'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x45', '@mqx') + 'p'](_0xed7f25, _0x572b62)) | _0x179134[_0x1533('0xa0', 'avK7') + 'p'](_0x5bc269, _0x572b62);\n" +
"\t\t\treturn _0x179134[_0x1533('0x5e', 'avK7') + 'V'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
"\t\t}\n" +
"\t\tfunction _0x496785(_0x583c97) {\n" +
"\t\t\treturn _0x583c97 < 0x14 ? 0x5a827999: _0x583c97 < 0x28 ? 0x6ed9eba1: _0x583c97 < 0x3c ? -0x70e44324: -0x359d3e2a;\n" +
"\t\t}\n" +
"\t\tvar _0x45b840 = _0x179134[_0x1533('0x39', 'R#BF') + 'R'](_0xde630f, _0x1a5a87);\n" +
"\t\tvar _0x53258d = new Array(0x50);\n" +
"\t\tvar _0xc86ce6 = 0x67452301;\n" +
"\t\tvar _0x3ee50e = -0x10325477;\n" +
"\t\tvar _0xdb1008 = -0x67452302;\n" +
"\t\tvar _0xe99bcb = 0x10325476;\n" +
"\t\tvar _0x56c860 = -0x3c2d1e10;\n" +
"\t\tfor (var _0x4c314c = 0x0; _0x4c314c < _0x45b840[_0x1533('0x46', '\$qIi') + 'th']; _0x4c314c += 0x10) {\n" +
"\t\t\tvar _0x41e8e8 = _0xc86ce6;\n" +
"\t\t\tvar _0x1b7277 = _0x3ee50e;\n" +
"\t\t\tvar _0x30f7c3 = _0xdb1008;\n" +
"\t\t\tvar _0x279c95 = _0xe99bcb;\n" +
"\t\t\tvar _0xf6e888 = _0x56c860;\n" +
"\t\t\tfor (var _0x77e31b = 0x0; _0x179134[_0x1533('0x34', 'Y8u@') + 'n'](_0x77e31b, 0x50); _0x77e31b++) {\n" +
"\t\t\t\tif (_0x77e31b < 0x10) {\n" +
"\t\t\t\t\tif (_0x179134[_0x1533('0xa6', 'Mifp') + 'T'](_0x1533('0x3f', 'o&Q4') + 'G', _0x179134[_0x1533('0xa1', 'rqYb') + 'n'])) {\n" +
"\t\t\t\t\t\t_0x53258d[_0x77e31b] = _0x45b840[_0x4c314c + _0x77e31b];\n" +
"\t\t\t\t\t} else {\n" +
"\t\t\t\t\t\tsleepT = _0x179134[_0x1533('0x48', 'NNf8') + 'R'](parseInt, data['wt']) > ret[0x1] ? _0x179134[_0x1533('0x97', 'd[i)') + 'R'](parseInt, data['wt']) - ret[0x1] : 0x1f4;\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\t_0x53258d[_0x77e31b] = _0x50e067(_0x179134[_0x1533('0x96', 'rQf(') + 'b'](_0x179134[_0x1533('0xb0', 'Gfc!') + 'b'](_0x53258d[_0x179134[_0x1533('0xa5', 'K)PG') + 'u'](_0x77e31b, 0x3)], _0x53258d[_0x179134[_0x1533('0x51', 'K)PG') + 'f'](_0x77e31b, 0x8)]) ^ _0x53258d[_0x77e31b - 0xe], _0x53258d[_0x179134[_0x1533('0xaf', '(U[3') + 'f'](_0x77e31b, 0x10)]), 0x1);\n" +
"\t\t\t\t}\n" +
"\t\t\t\tt = _0x179134[_0x1533('0x18', 'GvA9') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x23', '\$qIi') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x8d', 'Mifp') + 'D'](_0x50e067, _0xc86ce6, 0x5), _0x179134[_0x1533('0x4c', 'xPGU') + 'm'](_0x526d3b, _0x77e31b, _0x3ee50e, _0xdb1008, _0xe99bcb)), _0x5be8c0(_0x5be8c0(_0x56c860, _0x53258d[_0x77e31b]), _0x179134[_0x1533('0xad', 'rqYb') + 'W'](_0x496785, _0x77e31b)));\n" +
"\t\t\t\t_0x56c860 = _0xe99bcb;\n" +
"\t\t\t\t_0xe99bcb = _0xdb1008;\n" +
"\t\t\t\t_0xdb1008 = _0x50e067(_0x3ee50e, 0x1e);\n" +
"\t\t\t\t_0x3ee50e = _0xc86ce6;\n" +
"\t\t\t\t_0xc86ce6 = t;\n" +
"\t\t\t}\n" +
"\t\t\t_0xc86ce6 = _0x5be8c0(_0xc86ce6, _0x41e8e8);\n" +
"\t\t\t_0x3ee50e = _0x5be8c0(_0x3ee50e, _0x1b7277);\n" +
"\t\t\t_0xdb1008 = _0x5be8c0(_0xdb1008, _0x30f7c3);\n" +
"\t\t\t_0xe99bcb = _0x5be8c0(_0xe99bcb, _0x279c95);\n" +
"\t\t\t_0x56c860 = _0x5be8c0(_0x56c860, _0xf6e888);\n" +
"\t\t}\n" +
"\t\treturn _0x179134[_0x1533('0x86', 'Gfc!') + 'A'](_0x179134[_0x1533('0x44', 'DDSy') + 'X'](_0x179134[_0x1533('0x8e', 'xPGU') + 'g'](_0x225a28, _0xc86ce6), _0x179134[_0x1533('0x77', 'n*Rs') + 'g'](_0x225a28, _0x3ee50e)) + _0x225a28(_0xdb1008) + _0x225a28(_0xe99bcb), _0x179134[_0x1533('0xd', 'DDSy') + 'g'](_0x225a28, _0x56c860));\n" +
"\t}\n" +
"\tfunction go(_0x394549) {\n" +
"\t\tvar _0x33d0d6 = {};\n" +
"\t\t_0x33d0d6[_0x1533('0x2b', 'sahq') + 'H'] = _0x1533('0x14', 'rQf(') + _0x1533('0x15', 'sahq');\n" +
"\t\t_0x33d0d6[_0x1533('0xab', 'dgE&') + 'H'] = function(_0x23f209, _0x4ed415) {\n" +
"\t\t\treturn _0x23f209 < _0x4ed415;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x5c', 'xPGU') + 'g'] = function(_0x39bd0a, _0x4d5764) {\n" +
"\t\t\treturn _0x39bd0a != _0x4d5764;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x37', 'y*wC') + 'N'] = function(_0x3c9a26, _0x109b89) {\n" +
"\t\t\treturn _0x3c9a26 + _0x109b89;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x63', 'gkl*') + 'z'] = function(_0x584935, _0x58347c) {\n" +
"\t\t\treturn _0x584935 + _0x58347c;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x21', 'ZaCV') + 'z'] = function(_0x3fc795, _0x6962f6) {\n" +
"\t\t\treturn _0x3fc795 + _0x6962f6;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x27', '(cNt') + 'I'] = function(_0x303446, _0x3388fb) {\n" +
"\t\t\treturn _0x303446 == _0x3388fb;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x6b', 'y*wC') + 't'] = function(_0x5cb515, _0x31e7f3) {\n" +
"\t\t\treturn _0x5cb515(_0x31e7f3);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x75', 'Rh@#') + 'E'] = function(_0x529d8e, _0xc1423a) {\n" +
"\t\t\treturn _0x529d8e + _0xc1423a;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x62', 'rQf(') + 'a'] = function(_0x453505, _0x50cd56) {\n" +
"\t\t\treturn _0x453505 + _0x50cd56;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x2e', 'NNf8') + 'B'] = _0x1533('0x28', ']*f@') + _0x1533('0x57', 'Kp2R') + '\\x20/';\n" +
"\t\t_0x33d0d6[_0x1533('0xe', 'Y8u@') + 'y'] = function(_0x25bcfd, _0x4b634e, _0x1bef22) {\n" +
"\t\t\treturn _0x25bcfd(_0x4b634e, _0x1bef22);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x92', 'J21I') + 'i'] = function(_0xccfc4d, _0x9a8620) {\n" +
"\t\t\treturn _0xccfc4d > _0x9a8620;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x11', '@H8e') + 'n'] = function(_0x1b9fc7, _0x7d3ce) {\n" +
"\t\t\treturn _0x1b9fc7(_0x7d3ce);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x1b', 'N#(C') + 'O'] = _0x1533('0x7', 'N#(C') + '失败';\n" +
"\t\tvar _0x2eda7d = _0x33d0d6;\n" +
"\t\tfunction _0x29e502() {\n" +
"\t\t\tvar _0x24a3c1 = window[_0x1533('0x35', 'ad*k') + _0x1533('0x5d', 'Y8u@') + 'r'][_0x1533('0x19', 'avK7') + _0x1533('0x7d', 'cJ3U') + 't'],\n" +
"\t\t\t_0x5f1192 = [_0x2eda7d[_0x1533('0x24', '(bZ%') + 'H']];\n" +
"\t\t\tfor (var _0x2d8686 = 0x0; _0x2eda7d[_0x1533('0x1a', 'avK7') + 'H'](_0x2d8686, _0x5f1192[_0x1533('0x3b', 'Y8u@') + 'th']); _0x2d8686++) {\n" +
"\t\t\t\tif (_0x2eda7d[_0x1533('0x2f', 'y*wC') + 'g'](_0x24a3c1[_0x1533('0x76', 'y*wC') + _0x1533('0x33', 'ZaCV')](_0x5f1192[_0x2d8686]), -0x1)) {\n" +
"\t\t\t\t\treturn !! [];\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\tif (window[_0x1533('0x41', 'dgE&') + _0x1533('0x6e', 'dgE&') + _0x1533('0x85', 'NNf8')] || window[_0x1533('0x30', 'o&Q4') + _0x1533('0x38', 'OhQr')] || window[_0x1533('0xc', 'OhQr') + _0x1533('0x74', 'qRqa')] || window[_0x1533('0x25', '(bZ%') + _0x1533('0x4f', 'avK7') + 'r'][_0x1533('0xa2', 'xPGU') + _0x1533('0x53', ')T5g') + 'r'] || window[_0x1533('0xaa', 'cJ3U') + _0x1533('0x6', 'K#@S') + 'r'][_0x1533('0x8f', 'rqYb') + _0x1533('0xac', '(bZ%') + _0x1533('0x9', 'Rh@#') + _0x1533('0x4d', 'rQf(') + 'e'] || window[_0x1533('0x31', 'N#(C') + _0x1533('0x79', 'dgE&') + 'r'][_0x1533('0x4b', 'ksVw') + _0x1533('0x10', 'gkl*') + _0x1533('0xa8', 'd[i)') + _0x1533('0x89', '\$qIi') + _0x1533('0x84', 's4Dp')]) {\n" +
"\t\t\t\treturn !! [];\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\tif (_0x29e502()) {\n" +
"\t\t\treturn;\n" +
"\t\t}\n" +
"\t\tvar _0x1642ef = new Date();\n" +
"\t\tfunction _0x1a1fd9(_0x55baea, _0x579edb) {\n" +
"\t\t\tvar _0x551ac6 = _0x394549[_0x1533('0x4a', 'y*wC') + 's'][_0x1533('0x71', 'K#@S') + 'th'];\n" +
"\t\t\tfor (var _0x54ccec = 0x0; _0x2eda7d[_0x1533('0x6f', 'dFiI') + 'H'](_0x54ccec, _0x551ac6); _0x54ccec++) {\n" +
"\t\t\t\tfor (var _0x562c2d = 0x0; _0x2eda7d[_0x1533('0x9b', '[H[u') + 'H'](_0x562c2d, _0x551ac6); _0x562c2d++) {\n" +
"\t\t\t\t\tvar _0x1b8d8c = _0x2eda7d[_0x1533('0x80', '[H[u') + 'N'](_0x2eda7d[_0x1533('0x3a', 'w*Nn') + 'z'](_0x2eda7d[_0x1533('0x8b', 'cJ3U') + 'z'](_0x579edb[0x0], _0x394549[_0x1533('0x32', 'R#BF') + 's'][_0x1533('0x82', '@mqx') + 'tr'](_0x54ccec, 0x1)), _0x394549[_0x1533('0x1f', '*hmq') + 's'][_0x1533('0x47', '(cNt') + 'tr'](_0x562c2d, 0x1)), _0x579edb[0x1]);\n" +
"\t\t\t\t\tif (_0x2eda7d[_0x1533('0x6a', '(bZ%') + 'I'](_0x2eda7d[_0x1533('0x9d', 'Kp2R') + 't'](hash, _0x1b8d8c), _0x55baea)) {\n" +
"\t\t\t\t\t\treturn [_0x1b8d8c, new Date() - _0x1642ef];\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\tvar _0x1e73b9 = _0x2eda7d[_0x1533('0x36', 'sahq') + 'y'](_0x1a1fd9, _0x394549['ct'], _0x394549[_0x1533('0x2d', '\$qIi')]);\n" +
"\t\tif (_0x1e73b9) {\n" +
"\t\t\tvar _0x3b1427;\n" +
"\t\t\tif (_0x394549['wt']) {\n" +
"\t\t\t\t_0x3b1427 = _0x2eda7d[_0x1533('0x1', 'zx(Q') + 'i'](parseInt(_0x394549['wt']), _0x1e73b9[0x1]) ? _0x2eda7d[_0x1533('0x54', 'avK7') + 'n'](parseInt, _0x394549['wt']) - _0x1e73b9[0x1] : 0x1f4;\n" +
"\t\t\t} else {\n" +
"\t\t\t\t_0x3b1427 = 0x5dc;\n" +
"\t\t\t}\n" +
"\t\t\t_0x2eda7d[_0x1533('0x72', 'qRqa') + 'y'](setTimeout,\n" +
"\t\t\tfunction() {\n" +
"\t\t\t\tdocument[_0x1533('0x20', 'rQf(') + 'ie'] = _0x2eda7d[_0x1533('0x73', 'rQf(') + 'E'](_0x2eda7d[_0x1533('0x3c', 'o&Q4') + 'a'](_0x394549['tn'] + '=' + _0x1e73b9[0x0] + (_0x1533('0x8a', 'ksVw') + _0x1533('0x1e', 'GvA9') + '='), _0x394549['vt']), _0x2eda7d[_0x1533('0x8c', 'Gfc!') + 'B']);\n" +
"\t\t\t\tlocation[_0x1533('0x7a', 'sahq')] = location[_0x1533('0x64', 'd[i)') + _0x1533('0x81', '(U[3')] + location[_0x1533('0x94', 'sahq') + 'ch'];\n" +
"\t\t\t},\n" +
"\t\t\t_0x3b1427);\n" +
"\t\t} else {\n" +
"\t\t\talert(_0x2eda7d[_0x1533('0x7b', ')T5g') + 'O']);\n" +
"\t\t}\n" +
"\t};\n" +
"\tgo({\n" +
"\t\t\"bts\": [\"1618915727.945|0|HM5\", \"VoVNIPqcrvFO13woIZK2RY%3D\"],\n" +
"\t\t\"chars\": \"ZDUZR0cDjTEXznMkfgavFO\",\n" +
"\t\t\"ct\": \"3049e28877124ad21ca0db55e3af95ff316ceef6\",\n" +
"\t\t\"ha\": \"sha1\",\n" +
"\t\t\"tn\": \"__jsl_clearance\",\n" +
"\t\t\"vt\": \"3600\",\n" +
"\t\t\"wt\": \"1500\"\n" +
"\t})"
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("javascript");
engine.eval(str);
//获取对象
System.out.println();
}
def getJsContent(def page){
ScriptEngineManager manager = new ScriptEngineManager();
//得到脚本引擎
ScriptEngine engine = manager.getEngineByName("javascript");
//处理加密js
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
try {
//得到解密后的js
js = js.replace("document.cookie","var cookie")
js = js.replace("location.href=location.pathname+location.search", "")
engine.eval(js);
def v = engine.get("cookie")
return v.toString().substring(0, v.toString().indexOf(";"))
} catch (ScriptException e) {
e.printStackTrace();
}
return null
}
def getJsContent2(def page){
ScriptEngineManager manager = new ScriptEngineManager();
//得到脚本引擎
ScriptEngine engine = manager.getEngineByName("javascript");
//处理加密js
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
try {
//得到解密后的js
//得到解密后的js
String result = (String) engine.eval(js);
result = result.substring(result.indexOf("var cd"),result.indexOf("dc+=cd;")+7);
result = result.replaceAll("document*.*toLowerCase\\(\\)", "'x'");
//得到cookie
String jsl = (String) engine.eval(result);
return jsl;
} catch (ScriptException e) {
e.printStackTrace();
}
return null
}
/**
* 通过破解动态JavaScript脚本,
* 获取cookie名为 __jsl_clearance的值
*
* @param body 相应内容(一般为第一次请求获取到的动态js字符串)
* @return cookie名为 __jsl_clearance的值
*/
private static String getJslClearance(String body) {
//V8:谷歌开源的运行JavaScript脚本的库. 参数:globalAlias=window, 表示window为全局别名,
// 告诉V8在运行JavaScript代码时, 不要从代码里找window的定义.
V8 runtime = V8.createV8Runtime("window");
//将第一次请求pdf资源时获取到的字符串提取成V8可执行的JavaScript代码
body = body.trim()
.replace("<script>", "")
.replace("</script>", "")
.replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))",
"y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})");
//用V8执行该段代码获取新的动态JavaScript脚本
String result = runtime.executeStringScript(body);
//获取 jsl_clearance 的第一段, 格式形如: 1543915851.312|0|
String startStr = "document.cookie='";
int i1 = result.indexOf(startStr) + startStr.length();
int i2 = result.indexOf("|0|");
String cookie1 = result.substring(i1, i2 + 3);
/*
获取 jsl_clearance 的第二段,格式形如: DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D。
主要原理是: 新的动态JavaScript脚本是为浏览器设置cookie, 且cookie名为__jsl_clearance
其中第一段值(格式形如:1543915851.312|0|)已经明文写好, 用字符串处理方法即可获取.
第二段则是一段JavaScript函数, 需要有V8运行返回,
该函数代码需要通过一些字符串定位, 提取出来, 交给V8运行.
*/
startStr = "|0|'+(function(){";
int i3 = result.indexOf(startStr) + startStr.length();
int i4 = result.indexOf("})()+';Expires");
String code = result.substring(i3, i4).replace(";return", ";");
String cookie2 = runtime.executeStringScript(code);
/*
拼接两段字符串, 返回jsl_clearance的完整的值.
格式形如: 1543915851.312|0|DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D
*/
return cookie1 + cookie2;
}
/**
* 将HTTP响应体转换为字符串返回
*
* @param response HTTP响应
* @return 响应体的字符串形式
* @throws IOException IO异常
*/
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
}
static String getJsluid(HttpResponse response) {
Header header = response.getFirstHeader("set-cookie");
String[] split = header.getValue().split(";");
for (String s : split) {
if (s.contains("__jsluid")) {
return s.trim();
}
}
return "";
}
def fristCrawl(){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Cache-Control","max-age=0")
// headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Accept-Encoding","gzip, deflate")
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
void process(Object args) {
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
for(def key : companyKeys){
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
def s = crawl(url, queryBody);
if(s){
print s
parser(s);
}
println s
}
}
def parser(content){
def document = Jsoup.parse(content);
if(document){
def detailUrls = []
Elements elements = JQueryUtils.select([".search_list_item"], document)
if(elements && elements.size()>0){
for(def el : elements){
detailUrls << el.attr("href")
}
}
}
}
def static crawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Content-Length","226")
headers.put("Cache-Control","max-age=0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Origin","http://www.gsxt.gov.cn")
headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
// headers.put("Accept-Encoding","gzip, deflate")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
districtMap.putAll(detailMap)
}
}
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
while(flag){
println "开始加载解析数据......"
@@ -116,7 +116,7 @@ class DealPriceMathBatch extends BaseBatch{
count++;
}catch (Exception e){
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.getMessage()
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.printStackTrace()
}finally {
latch.countDown()
}
@@ -442,16 +442,21 @@ class DealPriceMathBatch extends BaseBatch{
if(regionMap){
regionMap.each{k,v->
def arrays = k.split("###")
if(arrays[1] && arrays[1]==keyword){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
try{
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
}
}catch(Exception e){
e.printStackTrace()
}
}
}
}else{//无区域
cityMap.each{r,dMap->
dMap.each{k,v->
def arrays = k.split("###")
if(arrays[1] && arrays[1]==keyword){
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
}
}
@@ -596,10 +601,13 @@ class DealPriceMathBatch extends BaseBatch{
}
def getOpenApiMatch(def city, def region, def name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
if(name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
}
}
return null;
}
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
houseLoudng.fitment_cost = ret.fitment_charge
houseLoudng.car_port = ret.carport
houseLoudng.house_height = ret.house_hight
houseLoudng.worry_facility" : null,
houseLoudng.fitment_age = ret.fitment_age
houseLoudng.lighting_ventilation" : null,
houseLoudng.build_type = ret.build_type
houseLoudng.house_property" : null,
houseLoudng.noise" : null,
houseLoudng.sight" : null,
houseLoudng.fitment_rank" : null,
houseLoudng.sideset" : null,
// houseLoudng.worry_facility" : null,
// houseLoudng.fitment_age = ret.fitment_age
// houseLoudng.lighting_ventilation" : null,
// houseLoudng.build_type = ret.build_type
// houseLoudng.house_property" : null,
// houseLoudng.noise" : null,
// houseLoudng.sight" : null,
// houseLoudng.fitment_rank" : null,
// houseLoudng.sideset" : null,
houseLoudng.district_no = district._id
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
@@ -37,7 +37,7 @@ class HouseExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -352,7 +352,7 @@ class HouseExportBatch extends BaseBatch{
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-2020*", queryBuilder);
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "latest_seawater", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
@@ -4,13 +4,13 @@ import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class AddEsHouseField {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -9,10 +9,9 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
@@ -63,7 +62,7 @@ class AddThirdPriceDistrict {
def province = null
def resAddrMap = [:]
def filterEstateTypeMap = [:]
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,11 +1,11 @@
package com.fangjia.script.gj
import com.esms.common.util.CommonUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import com.huaban.analysis.jieba.JiebaSegmenter
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -485,7 +485,7 @@ class CrawlDataMatch {
}
def findAllAddress(def cityName){
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
File diff suppressed because it is too large Load Diff
@@ -1,12 +1,12 @@
package com.fangjia.script.gj
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class EsSearchUtils {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -8,10 +8,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -20,7 +19,7 @@ class FZDataExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -42,10 +42,10 @@ class GaoDeMapCrawl {
def updateErrorLists = []
def flagLists = []
def index = 0
def prefix = "road_"
def prefix = "district_"
// def categorys = ["141202", "141203"]//中学,小学
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
def categorys = ["190301"]//道路相关
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def categorys = ["190301"]//道路相关
static main(args) {
def task = new GaoDeMapCrawl();
@@ -109,8 +109,8 @@ class GaoDeMapCrawl {
def run(){//,"西安","重庆"
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"]
def citys = CommonUtils.findAllCity()
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
def cityLations = findAllLation(city);
@@ -198,9 +198,11 @@ class GaoDeMapCrawl {
if(m.rets){
processData(m.rets);
location.count = m.rets.size()
location.content = m.toString()
flagLists.add(location)
}else{
location.count = 0
location.content = m.toString()
flagLists.add(location)
}
@@ -336,7 +338,7 @@ class GaoDeMapCrawl {
for(def location : updateLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","完成").append("m_date", new Date())
def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -358,7 +360,7 @@ class GaoDeMapCrawl {
for(def location : flagLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date())
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -382,7 +384,7 @@ class GaoDeMapCrawl {
for(def location : updateErrorLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","失败").append("m_date", new Date())
def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -0,0 +1,131 @@
package com.fangjia.script.gj
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.lucene.queryparser.classic.QueryParser
class LogStat {
static main(args) {
def task = new LogStat()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d ->
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
if(d['A']){
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(district){
def startDate = DateUtil.addMonth(new Date(), -6)
def start = 20200901//startDate.replace("-","").toInteger()
def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def count = findHouses(d['A'], d['B'], d['C'], alias, start, end)
d['D'] = count
println(d)
}
}
},0)
}
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
def count = 0
ElasticUtils.executeSqlQuery(sql,{data->
count = data.c
})
return count
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
// def findTestDatas(def city, def value) {
// def q = new BasicDBObject()
// q.put("user_id", "1007751")
// q.put("parameter_map.city", city)
// q.put("request_uri", "/property/evaluateForTest")
//
// def keys = new BasicDBObject()
// keys.put("res_map",1)
// def cursor = LOG.find(q)
// def count = 0
// println "总量:"+cursor.count()
// def datas = []
// for(def d : cursor){
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
// def resMap = d.res_map.result
// if(resMap.calcType==value){
// def data = []
// def rowMap = [:]
// rowMap.city = resMap.city
// rowMap.region = resMap.district
// rowMap.name = resMap.name
// rowMap.calDate = d.parameter_map.calDate
// data << rowMap
// }
// }
// }
// return datas;
// }
// for(def i=0;i<datas.size();i++){
// def d = datas[i];
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
//
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// if(district){
// def startDate = DateUtil.addMonth(calDate, -6)
// def start = startDate.replace("-","").toInteger()
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
// def alias = []
// if(district.alias){
// if(!district.alias.contains(district.name)){
// district.alias << district.name
// }
// alias = district.alias
// }else{
// alias << district.name
// }
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
// d['G'] = count
// }
}
@@ -7,11 +7,10 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -27,7 +26,7 @@ class NoHouseDistrictStatBatch extends BaseBatch{
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -3,14 +3,14 @@ package com.fangjia.script.gj
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class RoomStat {
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,30 +1,25 @@
package com.fangjia.script.gj
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.zip.GZIPInputStream
import static org.elasticsearch.index.query.QueryBuilders.idsQuery
class TradeDataMatch {
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def final AtomicInteger count_ = new AtomicInteger(0)
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
File diff suppressed because one or more lines are too long
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-02-01"
static def statDate = "2021-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -0,0 +1,175 @@
package com.fangjia.script.gj.address
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class AddressParser {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddressParser();
task.run()
// task.gaoDeAddress()
}
def gaoDeAddress(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match2(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def run(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def match(id, city, region, address, name){
def locationMap = null;
// if(region && !name.contains(region)){
// locationMap = matchLocation(city, region + name, "name")
// }else{
locationMap = matchLocation(city + region, name, "name")
// }
// if(!locationMap){
// locationMap = matchLocation(city, address, "address")
// }
if(locationMap){
DBObject data = new BasicDBObject()
data.put("school_lat1", locationMap.glat)
data.put("school_lng1", locationMap.glng)
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def match2(id, city, region, address, name){
def locationMap = null;
locationMap = matchLocation2(city + region, name, "name")
if(!locationMap){
locationMap = matchLocation2(city, address, "address")
}
if(locationMap){
DBObject data = new BasicDBObject()
data.put("gaode_name", locationMap.name)
data.put("gaode_address", locationMap.address)
if(locationMap.location){
def arrs = locationMap.location.split(",")
data.put("school_lat", arrs[1].toDouble())
data.put("school_lng", arrs[0].toDouble())
}
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def matchLocation(city, address, cat){
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
def map = [:]
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def datas = geocodes.get(0)
if(datas.formatted_address && cat=="name"){
if(datas.formatted_address.contains(address)){
def glocation = datas.location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}else{
return null
}
}else{
def glocation = geocodes.get(0).location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}
}
return map
}
def matchLocation2(city, address, cat){
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
def map = [:]
if (gloJson && gloJson.get("pois")) {
def geocodes = gloJson.get("pois")
def datas = geocodes.get(0)
return datas
}
return null
}
def findAllDatas(){
def q = new BasicDBObject()
// q.put("school_name","苏州市娄葑实验小学")
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -0,0 +1,354 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class DistrictSchoolDataExport {
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def regionCodeMap = [:]
static main(args) {
def task = new DistrictSchoolDataExport()
// task.run()
task.megre()
}
def megre(){
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
def ids = []
for(def d : datas1){
ids << d['U']
}
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
for(def d : datas2){
def id = d['U']
if(!ids.contains(id)){
datas1 << d
}
}
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
}
def run(){
def city = "苏州"
getRegionCodeByCity(city)
getRegionCodeByCity("合肥")
getRegionCodeByCity("西安")
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts = []
districts.addAll(districts0)
districts.addAll(districts1)
districts.addAll(districts2)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = []
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
def i = 0
for(def k=0;k<districts.size();k++){
final def dist = districts.get(k)
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
if(primarySchools || midSchools){
def info1 = null;
def info2 = null;
def distance1 = 0
def distance2 = 0
if(primarySchools && dist.lng){
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
if(info1 && info1.school_lat2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
if(distance1>3000){
info1 = null
}
}else{
info1 = null
}
}
if(midSchools) {
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
if(info2 && info2.school_lng2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
if(distance2>3000){
info2 = null
}
}else{
info2 = null
}
}
if(info1|| info2){
def row = []
row << cityCodeMap.get(dist.city)
row << regionCodeMap.get(dist.city +"-"+dist.region)
row << dist.name
row << dist.lng/10000000000.0
row << dist.lat/10000000000.0
row << dist.address
if(info1){
row << info1.school_name
row << info1.school_address!=null?info1.school_address:''
row << info1.attribute!=null?info1.attribute:''
row << info1.school_lng2!=null?info1.school_lng2:''
row << info1.school_lat2!=null?info1.school_lat2:''
// row << distance1
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
if(info2){
row << info2.school_name
row << info2.school_address!=null?info2.school_address:''
row << info2.attribute!=null?info2.attribute:''
row << info2.school_lng2!=null?info2.school_lng2:''
row << info2.school_lat2!=null?info2.school_lat2:''
// row << distance2
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
row << ""
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
row << "fjw"
row << dist._id.toString()
row << "对口"
rets << row
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
}
def selectSchool(lng, lat, def schools){
def minSchool = null
def level0 = []//国家
def level1 = []//省重点
def level2 = []//市重点
def level3 = []//区重点
def level4 = []//普通
def level5 = []//其它
for(def s : schools){
if(s.attribute){
if(s.attribute.contains("国家重点")){
level0 << s
}else if(s.attribute.contains("省重点")){
level1 << s
}else if(s.attribute.contains("市重点")){
level2 << s
}else if(s.attribute.contains("区重点")){
level3 << s
}else if(s.attribute.contains("普通")){
level4 << s
}else{
level5 << s
}
}else{
level5 << s
}
}
if(level0){
if(level0.size()==1){
minSchool = level0.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level0)
}
}else if(level1){
if(level1.size()==1){
minSchool = level1.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level1)
}
}else if(level2){
if(level2.size()==1){
minSchool = level2.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level2)
}
}else if(level3){
if(level3.size()==1){
minSchool = level3.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level3)
}
}else if(level4){
if(level4.size()==1){
minSchool = level4.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level4)
}
}else if(level5){
if(level5.size()==1){
minSchool = level5.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level5)
}
}
def rets = []
if(minSchool){
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
}
if(rets.size()>0){
for(def d : rets){
if(!minSchool.school_address){
minSchool.school_address = d.address
}
if(!minSchool.attribute){
minSchool.attribute = d.attribute
}
}
}else{
if(minSchool){
if(minSchool.school_map_lng){
minSchool.school_lng2 = minSchool.school_map_lng
minSchool.school_lat2 = minSchool.school_map_lat
}else if(minSchool.school_lng1){
minSchool.school_lng2 = minSchool.school_lng1
minSchool.school_lat2 = minSchool.school_lat1
}else if(minSchool.school_lng){
minSchool.school_lng2 = minSchool.school_lng
minSchool.school_lat2 = minSchool.school_lat
}
}
}
return minSchool;
}
def findSchoolInfo(def city, def region, def schoolName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", schoolName)
def dbcursor = school_col.find(q)
def rets = []
for(def obj : dbcursor){
rets << obj
}
return rets
}
def selectMinDistanceSchool(def lat, def lng, def schools){
def minSchool = null;
def minDistance = Integer.MAX_VALUE
for(def s1 : schools){
def lng2 = null
def lat2 = null
if(s1.school_map_lng){
lng2 = s1.school_map_lng
lat2 = s1.school_map_lat
}else if(s1.school_lng1){
lng2 = s1.school_lng1
lat2 = s1.school_lat1
}else if(s1.school_lng){
lng2 = s1.school_lng
lat2 = s1.school_lat
}
if(lat2){
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
println s1.district_name +" "+ s1.school_name+" "+distance
if(distance<minDistance){
minDistance = distance
minSchool = s1
minSchool.school_lng2 = lng2
minSchool.school_lat2 = lat2
}
}
}
return minSchool
}
def findPrimarySchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def findMiddleSchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def getRegionCodeByCity(def city){
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
for(def d : datas){
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
}
}
}
@@ -1,24 +1,17 @@
package com.fangjia.script.gj.ccb
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet
import java.text.SimpleDateFormat
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -26,12 +19,11 @@ import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.script.util.FangJiaAreaMapUtils
class HouseExport {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -0,0 +1,48 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class RegionPriceExport {
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionPriceExport();
task.run()
}
def run(){
def months = [202010,202011,202012,202101,202102,202103]
def regions = CommonUtils.getAllRegions("福州","region")
def rets = []
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
regions.each{region,values->
def data = []
data << "福州"
data << region
for(def month :months){
def dbo = findPrice("福州", region, month);
if(dbo){
data << dbo.avg_price
}else{
data << ""
}
}
rets << data
}
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
}
def findPrice(city, region, month){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("date", month)
def dbo = REGION_SELL_STAT.findOne(q)
return dbo
}
}
@@ -5,16 +5,14 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
class ExportData {
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -5,16 +5,11 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.*
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -27,7 +22,7 @@ class DistrictWash {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
static def totalCount = new AtomicLong()
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -0,0 +1,182 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.core.util.BaiDuApiUtils
import com.mongodb.BasicDBObject
import com.fangjia.script.util.DbUtils
class FZloationUpdate {
static main(args) {
def task = new FZloationUpdate();
task.run()
}
def run(){
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
def index = 0
def locationMap = getMoreSameLocationDist(districts)
def total = 0
locationMap.each {k,v->
total = updateLocation(v, total);
println "---------------------------------------------------------------------------------------------------------------"
}
}
def updateLocation(def lists, def total){
for(def dist : lists){
def addrDistMap = getLocation(dist, dist.address)
def nameDistMap = getLocation(dist, dist.region + dist.name)
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
if(addrDistMap){
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
def baidu = null
def lat = null
def lng = null
def upldateFlag = ""
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
lat = addrDistMap.lat
lng = addrDistMap.lng
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
upldateFlag = "Address"
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
lng = nameDistMap.lng
lat = nameDistMap.lat
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
upldateFlag = "Name"
}
if(baidu){
total++
def udata = new BasicDBObject()
udata.put("m_date", new Date())
Long baiduLat = baidu.y*10000000000L
Long baiduLng = baidu.x*10000000000L
udata.put("lat2", baiduLat)
udata.put("lng2", baiduLng)
Long lat0 = lat*10000000000L
Long lng0 = lng*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
}
}
}
}
return total
}
def getMoreSameLocationDist(def districts){
def locationMap = [:]
for(def i=0;i<districts.size();i++){
def dist = districts[i]
def lat = dist.lat/10000000000.0;
def lng = dist.lng/10000000000.0;
def locationKey = lng + "-" + lat
def list = locationMap.get(locationKey)
if(!list){
list = []
locationMap.put(locationKey, list);
}
list << dist
}
def resMap = [:]
locationMap.each{k,v->
if(v.size()>=2){
v.sort{ a,b->
return a.name.size().compareTo(b.name.size())
}
resMap.put(k,v)
}
}
def lastRetMaps = [:]
resMap.each{k,v->
def swname = null
def count = 0
for(def i=0;i<v.size();i++){
def d = v[i]
if(i==0){
if(d.name.contains("期")||d.name.contains("区")){
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
}else{
swname = d.name
}
}else{
if(d.name.contains(swname)||swname.contains(d.name)){
count++
}
}
}
if(count+1!=v.size()){
lastRetMaps.put(k,v)
}
}
return lastRetMaps;
}
def deleteRegion(def s, def region){
def key = ""
if(s.indexOf(region)>-1){
if(s.indexOf(region+"区")>-1){
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
}else if(s.indexOf(region+"县")>-1){
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
}else if(s.indexOf(region+"市")>-1){
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
}
}else{
key = s;
}
return key
}
def getLocation(dist, addr){
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(dist.city, addr);
if(reps && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
if(location){
def locMap = location.split(",")
def lat = dist.lat/10000000000.0;
def lng = dist.lng/10000000000.0;
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
rMap.distance = distance
rMap.level = level
rMap.lat = locMap[1].toDouble()
rMap.lng = locMap[0].toDouble()
rMap.keyword = map.formatted_address
rMap.location = location
rMap.region = map.district
return rMap;
}
}
}
return null;
}
}
@@ -0,0 +1,4 @@
package com.fangjia.script.gj.test
class FzLocationBatch {
}
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
// task.matchHouse();
// task.villa()
task.addressParse()
// task.addressParse()
task.deleteProcess()
}
def repeat(){
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
return floor;
}
def deleteProcess(){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
def addrList = []
for(def d : datas){
def addr = d['G']
addrList << addr
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
def addr = row['C']
if(addrList.contains(addr)){
row['R'] = "已调查"
}
},0);
}
}
@@ -0,0 +1,3 @@
package com.fangjia.script.gj.test
println "wwww"
@@ -18,8 +18,7 @@ import org.apache.commons.lang.StringUtils
class AutocompleteDataUpdaterBatch extends BaseBatch {
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "192.168.0.51", 29009)
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "mongo.db.fangjia.com")
private static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
private static final DSCollection AUTOCOMPLETE_COLL = new DataStore("192.168.0.38", 27008).getCollection("fangjia.autocomplete")
private static final long C_DATE = DateUtil.formatToYYYYMMDDhhmmss(new Date()).toLong()
@@ -2,38 +2,28 @@ package com.fangjia.script.jiangnanblank
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet
import java.text.DecimalFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils;
import com.fangjia.script.util.DateUtils;
import com.fangjia.script.util.ExcelParser
import com.fangjia.script.common.DataHandleUtils;
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.*
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.io.FileUtils;
import org.bson.types.ObjectId
import com.fangjia.script.util.MongoUtils;
import com.mongodb.QueryOperators
import com.fangjia.script.util.OpenApiClient
import java.util.Map.Entry;
import net.sf.json.JSONObject
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import org.apache.commons.lang.StringUtils
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.text.DecimalFormat
import java.util.Map.Entry
import java.util.regex.Matcher
import java.util.regex.Pattern
class CZBlankDataExport {
@@ -62,7 +52,7 @@ class CZBlankDataExport {
def directionMap = [1:"ÄÏ",2:"±±",3:"Äϱ±",4:"¶«",5:"Î÷",6:"¶«Î÷",7:"¶«ÄÏ",8:"Î÷ÄÏ",9:"¶«±±",10:"Î÷±±"]
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -12,7 +12,6 @@ import com.jcraft.jsch.ChannelSftp
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
/**
* 江南银行线上数据交付
@@ -6,42 +6,27 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.CrawlCommandConsumerBatch
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.BatchUpdateException
import java.sql.ResultSet
import java.text.NumberFormat
import java.util.Map.Entry
import java.util.concurrent.BlockingQueue
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.LinkedBlockingDeque
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit
import java.util.regex.Pattern
@BatchOwner(Watcher.WeiLei)
class HousingOverviewExportBatch extends BaseBatch {
static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque<Runnable>())
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,11 +1,10 @@
package com.fangjia.script.jusfoun
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.GroovyRowResult
@@ -18,9 +17,7 @@ import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import redis.clients.jedis.JedisPool
import redis.clients.jedis.JedisPoolConfig
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.*
import java.text.DateFormat
@@ -56,7 +53,7 @@ class JusfounCommon {
public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
private static JdbcDataSource dataSource = null
private static EsDataSource dataSource = null
private static Map<String, BasicDBObject> _map_seaweed = new HashMap<>();
private static Map<String, Map<String, DBObject>> mainCityMap = new HashMap<>();
@@ -390,7 +387,7 @@ class JusfounCommon {
return DataUtil.hanyuQuanpin(hanzi, '')
}
static JdbcDataSource getESDataSource() {
static EsDataSource getESDataSource() {
if (dataSource != null) {
return dataSource;
} else {
@@ -399,7 +396,7 @@ class JusfounCommon {
return dataSource;
}
try {
dataSource = new JdbcDataSource();
dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties();
@@ -625,13 +622,13 @@ class JusfounCommon {
String scrollID = null) {
try {
if (transportClient == null) {
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
transportClient = ElasticUtils.getClient("192.168.0.206");
}
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
} catch (Exception e) {
e.printStackTrace()
try {
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
transportClient = ElasticUtils.getClient("192.168.0.206");
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
} catch (Exception e1) {
e1.printStackTrace()
@@ -8,21 +8,13 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.sql.BatchUpdateException
import java.util.concurrent.LinkedBlockingDeque
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit
@BatchOwner(Watcher.WangWenYan)
class RentHouseExportBatch extends BaseBatch {
@@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
import org.apache.commons.math3.stat.descriptive.moment.Mean
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
import static com.mongodb.client.model.Filters.*
def List find_rent_house_prices(String city, String region, String name, Date start, Date end){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,20 +1,17 @@
package com.fangjia.script.log
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
@BatchOwner(Watcher.JiangYaHui)
class MongoDBLogDisMonitorBatch extends BaseBatch {
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
@@ -1,20 +1,18 @@
package com.fangjia.script.log
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
@BatchOwner(Watcher.JiangYaHui)
class MongoDBLogMonitorBatch extends BaseBatch {
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
@@ -10,7 +10,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.atomic.AtomicInteger
@@ -25,7 +26,7 @@ class DistrictHouseTypeAnalyzeBatch extends BaseBatch {
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final JdbcDataSource dataSource = new JdbcDataSource()
private static final EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -5,7 +5,7 @@ import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -55,7 +55,6 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
Map userInfo = getUserInfo()
def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
def endDate = DateUtil.truncateDay(new Date())
BasicDBObject query = new BasicDBObject()
query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
@@ -64,7 +63,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
long count = 0
long total = cursor.count()
print("total="+total)
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("192.168.0.206", "fangjia")
BulkProcessor bulkProcessor = ElasticUtils.createBulkProcessor("seawater.es.db.fangjia.com")
for(BasicDBObject log : cursor){
Map<String,Object> parameter_map = (Map) log.get("parameter_map")
if(parameter_map == null) {
@@ -128,7 +127,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
cmd.put("user_name", user.user_name)
cmd.put("company_name", user.company)
}
bulkProcessor.add(new IndexRequest("open_uri_log_normal", "_doc", log.get("_id").toString()).source(cmd))
bulkProcessor.add(new IndexRequest("open_uri_log_normal").id(log.get("_id").toString()).source(cmd))
println(cmd)
count++
if(count % 100 == 0){
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202101);
def count2 = findTotalCount(source, 202102);
def count1 = findTotalCount(source, 202102);
def count2 = findTotalCount(source, 202103);
println source +" "+count1 +" "+count2
}
}
@@ -0,0 +1,213 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.groovy.JsonSlurper
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class AddFangSchoolRegion {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddFangSchoolRegion();
//task.process()
//task.match()
task.addMapLocation()
}
def addMapLocation(){
def q = new BasicDBObject()
def dbcursor = district_school_col.find(q)
def index = 0
for(def d : dbcursor){
def data = findSchool(d.city, d.region, d.school_name)
if(data && data.location){
def location = data.location
if(location){
def udata = new BasicDBObject()
udata.put("school_map_lng", location[0])
udata.put("school_map_lat", location[1])
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
index ++
println index
}
}
def match(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
// if(!data.match_name){
println i +" "+data.district_name
matching(data);
// }
} catch (Exception e) {
println "Òì³£"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def matching(def it){
def rMap = matchDistrict(it.city, it.region, it.district_name)
if(!rMap){
rMap = matchDistrict(it.city, it.region, it.address)
}
if(rMap){
DBObject data = new BasicDBObject()
data.put("match_city", rMap.city)
data.put("match_region", rMap.region)
data.put("match_name", rMap.name)
data.put("match_address", rMap.address)
data.put("match_lat", rMap.lat)
data.put("match_lng", rMap.lng)
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
}
}
def matchDistrict(def city, def region, def address){
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dto = doc.result
if(dto && dto.searchAddress){
def ret = dto.matchResult
if(ret.match){
rMap.city = dto.city
rMap.region = dto.district
rMap.name = dto.name
rMap.lat = dto.lat
rMap.lng = dto.lng
rMap.id = dto.id
rMap.address = dto.address
}
}
}
}
return rMap;
}
def process(){
def datas = findNoRegionData();
findSimialData(datas);
}
def findAllDatas(){
def q = new BasicDBObject()
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
def findNoRegionData(){
def q = new BasicDBObject()
q.put("region", null)
def dbcursor = district_school_col.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findSimialData(datas){
for(def d : datas){
def obj = findThirdPriceData(d.city, d.url+"/")
if(!obj){
obj = findThirdPriceByName(d.city, d.district_name)
}
if(obj){
d.region = obj.region
if(!d.district_addrss){
d.district_address = obj.address
}
updateDistrictSchool(d);
}else{
println d.school_name+"" + d.district_name+"û²éµ½"
}
}
}
def findThirdPriceData(def city, def url){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("district_id", url)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def findThirdPriceByName(def city, def name){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("name", name)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def updateDistrictSchool(d){
DBObject data = new BasicDBObject()
data.put("region", d.region)
data.put("district_address", d.district_address)
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
}
def findSchool(city, region, school_name){
DBObject data = new BasicDBObject()
data.put("city", city)
data.put("region", region)
data.put("name", school_name)
return school_col.findOne(data)
}
}
@@ -0,0 +1,344 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.alibaba.fastjson.annotation.JSONField
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.httpclient.NameValuePair
import org.apache.commons.lang.StringUtils
import org.apache.http.HttpEntity
import org.apache.http.client.entity.UrlEncodedFormEntity
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpPost
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import java.nio.charset.Charset
import java.util.zip.GZIPInputStream
class GushuwangCrawl {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
static main(args) {
def task = new GushuwangCrawl();
task.process()
}
void process(Object args) {
// chaxuequCrawl()
suzhouCrawl();
}
def suzhouCrawl(){
def page = 1
def stop = false
def list = []
while(!stop){
def s = schoolList(page);
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.ok){
def datas = jsonObject.extra.list
if(datas){
list.addAll(datas)
page++
println page
}else{
stop = true
}
}else{
stop = true
}
}
}
if(list){
def schools = []
for(def d : list){
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = "苏州"
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(d.schoolname)
schoolData.name = d.schoolname
schoolData.address = d.adr
/**来源网站*/
schoolData.source = "苏州学区查询"
/**房源详情url*/
schools << schoolData
}
saveChaxuequSchool(schools)
}
}
def chaxuequCrawl(){
def page = 1
def stop = false
def schools = []
while(!stop){
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
println "内容:"+s
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject.code == 200){
def data = jsonObject.data
def totalPage = data.pages
if(data.list){
page++
if(page+1>totalPage){
stop = true;
}
schools.addAll(data.list)
}
}
}
}
print(schools.size());
def districtSchools = []
def schoolInfos = []
if(schools.size()>0){
def url = "https://51yongka.com/fapi/school/details"
for(def d : schools){
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
def s = chaxuequInfoCrawl(url, params)
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.code==200){
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
def districts = jsonObject.data.nearbyPropertyList
for(def dist : districts){
def dataMap = new BasicDBObject()
dataMap.city = jsonObject.data.city
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
dataMap.category = converCategory(jsonObject.data.schoolName)
dataMap.district_name = dist.name.trim()
dataMap.district_address = dist.address
dataMap.school_name = jsonObject.data.schoolName.trim()
dataMap.school_address = jsonObject.data.address
dataMap.source = "查学区房"
dataMap.url = url + "?"+params
dataMap.telephone = jsonObject.data.officialTelephone;
districtSchools << dataMap
}
}else{
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = jsonObject.data.city
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(jsonObject.data.schoolName)
schoolData.name = jsonObject.data.schoolName.trim()
schoolData.address = jsonObject.data.address
/**来源网站*/
schoolData.source = "查学区房"
/**房源详情url*/
schoolData.telephone = jsonObject.data.officialTelephone;
schoolInfos << schoolData
println jsonObject.data.schoolName +" 无周边楼盘"
}
}
println s;
}
}
println "小区:"+districtSchools.size()
// saveChaxuequDistrictSchool(districtSchools)
saveChaxuequSchool(schoolInfos)
}
def saveChaxuequSchool(schoolInfos){
for(def ds : schoolInfos){
def obj = findSchoolData(ds.city, ds.school_name);
if(!obj){
school_col.save(ds)
}
}
}
def saveChaxuequDistrictSchool(def districtSchools){
for(def ds : districtSchools){
def obj = findDistrictSchoolData(ds.city, ds.school_name);
if(!obj){
district_school_col.save(ds)
}
}
}
def findSchoolData(city, school_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
return school_col.findOne(q)
}
def findDistrictSchoolData(city, school_name, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
q.put("district_name", district_name)
return district_school_col.findOne(q)
}
def converCategory(def schoolName){
if(schoolName){
if(schoolName.contains("初中部")||schoolName.contains("中学")){
return "初中"
}else if(schoolName.contains("小学")){
return "小学"
}
}
return null
}
def static chaxuequInfoCrawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","51yongka.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","75")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
headers.put("Accept-Encoding","gzip, deflate, br")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
def static schoolList(page) {
def responseMap = [:]
String s = null
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","m.gusuwang.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","40")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
// headers.put("Accept-Encoding","gzip, deflate, br")
//params信息
Map<String, String> params = new HashMap<String, String>();
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
// //post请求
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
} catch (Exception e) {
e.printStackTrace()
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
outputStream.write(content.getBytes());
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
// byte[] data = new byte[1024];
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
// while (inputStream.read(data) != -1) {
// String s = new String(data, Charset.forName("utf-8"));
// sb.append(s);
// }
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
}
@@ -1,50 +1,24 @@
package com.fangjia.script.trade
///µ¼Èë¼Û¸ñÖ¸Êý
import com.fangjia.script.BaseBatch;
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
///导入价格指数
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.xssf.usermodel.XSSFCell
import org.apache.poi.xssf.usermodel.XSSFWorkbook
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.QueryOperators
import com.fangjia.script.util.DateUtils;
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.Mongo
import groovy.json.JsonSlurper
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.RandomUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.text.NumberFormat;
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.core.util.DBIterator
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.StringUtil;
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils;
import com.ibm.icu.text.SimpleDateFormat;
import com.fangjia.storehouse.DBIterator
import com.fangjia.storehouse.DBIterator;
import com.fangjia.storehouse.StorehouseDB;
import org.apache.commons.lang.math.RandomUtils;
@BatchOwner(Watcher.GaoJuan)
class ImportIndexStats extends BaseBatch{
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
@@ -61,7 +35,7 @@ class ImportIndexStats extends BaseBatch{
public static final def dingjiYear = 2016;
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final JdbcDataSource dataSource = new JdbcDataSource()
private static final EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,17 +1,12 @@
package com.fangjia.script.trade
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import com.mongodb.*
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -22,7 +17,7 @@ class TradeDealPriceStat {
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -3,7 +3,7 @@ package com.fangjia.script.usahouse
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
class AddressMatch {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,6 +1,6 @@
package com.fangjia.script.usahouse
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch
@@ -8,14 +8,11 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSON
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.select.Elements
@@ -425,6 +425,7 @@ class ABuYunUtil {
buffer.append(bf, 0, len);
}
} catch (IOException e) {
e.printStackTrace()
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
} finally {
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())
@@ -1,122 +1,169 @@
package com.fangjia.script.util
import org.apache.commons.lang3.SystemUtils
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchType
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.transport.TransportAddress
import org.elasticsearch.common.unit.ByteSizeUnit
import org.elasticsearch.common.unit.ByteSizeValue
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.sort.FieldSortBuilder
import org.elasticsearch.search.sort.SortOrder
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticUtils {
private static TransportClient client = null;
public static TransportClient getClient(def host){
if(client != null){
return client;
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
connectionProperties.setProperty("user","developer");
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
dataSource.setProperties(connectionProperties);
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
}
def certPath = ""
if(SystemUtils.IS_OS_WINDOWS){
certPath= "D:\\goojiaconfig\\seawater\\conf\\elastic-certificates.p12"
}else{
certPath = "/opt/elasticmongo/elastic-certificates.p12"
rs.getMetaData().getColumnCount()
while(rs.next()){
def row = [:]
for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
Settings.Builder builder = Settings.builder()
.put("cluster.name", "fangjia")
.put("client.transport.sniff", true)
builder.put("xpack.security.user", "developer" + ":" + "5e8edd851d2fdfbd7415232c67367cc3")
.put("xpack.security.transport.ssl.enabled", "true")
.put("xpack.security.transport.ssl.truststore.path", certPath)
.put("xpack.security.transport.ssl.keystore.path", certPath)
.put("xpack.security.transport.ssl.verification_mode", "certificate")
Settings settings = builder.build();
client = new PreBuiltXPackTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
return client
rs.close()
connection.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure, String type= "_doc"){
client = getClient(host);
BulkProcessor bulkProcessor = BulkProcessor.builder(
client,
new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId,
BulkRequest request) {
}
@Override
void afterBulk(long executionId,
BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId,
BulkRequest request,
Throwable failure) {
}
})
.setBulkActions(10000)
.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB))
.setFlushInterval(TimeValue.timeValueSeconds(5))
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10))
.build()
SearchResponse scrollResp = null;
int errorCount = 0;
def hits ;
def stop = false;
while(!stop){
try{
scrollResp = client.prepareSearch(indexName)
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
.setScroll(new TimeValue(600000))
.setQuery(qb)
.setSize(1000).get()
hits = scrollResp.getHits().getHits()
stop = true;
}catch(Exception e){
errorCount++;
if(errorCount>11){
stop = true;
}
println "错误:${errorCount} "+e.getMessage()
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 60);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
* @return
*/
static createBulkProcessor(String host, RestHighLevelClient client = null) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
};
if(!client){
client = getClient(host);
}
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
if(hits){
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
for (int k=0;k<hits.size();k++) {
final SearchHit hit = hits[k];
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
final SearchHit hit = searchHits[k];
executorService.submit(new Runnable() {
@Override
void run() {
@@ -124,96 +171,66 @@ class ElasticUtils {
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
// println(hits)
e.printStackTrace()
}
}
})
}
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
while(scrollResp.getHits().getHits().length != 0){
hits = scrollResp.getHits().getHits();
for (int k=0;k<hits.size();k++) {
final SearchHit hit = hits[k];
executorService.submit(new Runnable() {
@Override
void run() {
try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
// println(hits)
e.printStackTrace()
}
}
})
}
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
}
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
client = getClient(host);
SearchResponse scrollResp = null;
int errorCount = 0;
def hits ;
def stop = false;
while(!stop){
try{
scrollResp = client.prepareSearch(indexName)
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
.setScroll(new TimeValue(600000))
.setQuery(qb)
.setSize(10000)
.setScroll(TimeValue.timeValueMinutes(1))//设置 search context 维护1分钟的有效期
.get()
stop = true;
}catch(Exception e){
errorCount++;
if(errorCount>11){
stop = true;
}
println "错误:${errorCount} "+e.getMessage()
}
}
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def results = []
while(scrollResp.getHits() && scrollResp.getHits().getHits().length != 0){
for (SearchHit searchHit : scrollResp.getHits().getHits()) {
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
SearchHit searchHit = searchHits[k]
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet();
//当searchHits的数组为空的时候结束循环,至此数据全部读取完毕
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
//删除scroll
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollResp.getScrollId());
client.clearScroll(clearScrollRequest).actionGet();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
client.close()
return results;
}
}
@@ -1,84 +0,0 @@
package com.fangjia.script.util
import org.apache.commons.lang3.SystemUtils
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.transport.TransportAddress
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
import java.util.concurrent.TimeUnit
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticsearchHelper {
static getClient(String host, String clusterName = "fangjia"){
Settings.Builder builder = Settings.builder()
.put("cluster.name", clusterName)
.put("client.transport.sniff", false)
.put("client.transport.ping_timeout", "60s")
.put("xpack.security.user","developer:5e8edd851d2fdfbd7415232c67367cc3")
.put("xpack.security.transport.ssl.enabled", "true")
if(SystemUtils.IS_OS_WINDOWS){
builder
.put("xpack.security.transport.ssl.truststore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
.put("xpack.security.transport.ssl.keystore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
}else{
builder
.put("xpack.security.transport.ssl.truststore.path", "/opt/elasticmongo/elastic-certificates.p12")
.put("xpack.security.transport.ssl.keystore.path", "/opt/elasticmongo/elastic-certificates.p12")
}
builder.put("xpack.security.transport.ssl.verification_mode", "certificate").build()
Settings settings = builder.build()
return new PreBuiltXPackTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
}
/**
* 创建一个ES批处理器
* @param clusterName
* @param host
* @return
*/
static createBulkProcessor( String host, String clusterName){
TransportClient client = getClient(host, clusterName)
BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request, Throwable failure) {
failure.printStackTrace()
}
}).setBulkActions(10000).setConcurrentRequests(20).build()
return bulkProcessor
}
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor){
bulkProcessor.awaitClose(10, TimeUnit.MINUTES)
bulkProcessor.flush()
bulkProcessor.close()
}
}
@@ -1,19 +1,17 @@
package com.fangjia.script.util
import com.mongodb.DBCursor
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
class HouseUtils{
static main(args){
@@ -121,9 +121,9 @@ public class OpenApiClient {
params.put("suggestName", suggestName)
}
String content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8")
while (content == null) {
content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8")
println("${city} ${input} ${cat}程序调用异常")
Thread.sleep(6000)
}
@@ -1,19 +1,18 @@
package com.fangjia.script.vipsite
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.ElasticUtils
import com.mongodb.DBObject
import org.elasticsearch.action.search.SearchRequestBuilder
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchType
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.SearchHits
try {
TransportClient transportClient = ElasticsearchHelper.getClient("192.168.0.206");
TransportClient transportClient = ElasticUtils.getClient("192.168.0.206");
QueryBuilder dateRange_avg_price = QueryBuilders.rangeQuery("avg_price").lte(0);
QueryBuilder dateRange_total_price = QueryBuilders.rangeQuery("total_price").lte(0);
@@ -1,6 +1,6 @@
package com.fangjia.script.weilei
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.Bytes
import com.mongodb.DBCollection
@@ -8,7 +8,7 @@ import com.mongodb.DBCursor
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -16,7 +16,7 @@ import java.sql.SQLException
import java.sql.Statement
import java.text.SimpleDateFormat
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties();
File diff suppressed because it is too large Load Diff
@@ -1,31 +1,16 @@
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.CommandResult
import com.mongodb.DB
import com.mongodb.DBCollection
import com.mongodb.Mongo
import com.mongodb.client.MongoCollection
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.*;
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
@@ -43,7 +28,7 @@ cityList.each { def city ->
}
})
roots.each { def root ->
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,23 +1,12 @@
package com.fangjia.script.wenyan.address
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.util.QQKeyPool
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -44,7 +33,7 @@ import java.sql.Statement
// data << (k.split("_") as List)
//}
//ExcelUtils.write("D:\\ÎÞ¾ù¼ÛÊý¾Ý.xlsx", data, "½á¹û")
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,20 +1,20 @@
package com.fangjia.script.wenyan.avgPrice
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
def findEstateType(String city, String region, String name, List alias) {
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -10,17 +10,12 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.math3.ml.clustering.CentroidCluster
import org.apache.commons.math3.ml.clustering.DoublePoint
import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017)
def findHouse(BasicDBObject district){
@@ -54,7 +49,7 @@ def findHouse(BasicDBObject district){
sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ")
sqlBuffer.append(" ORDER BY p_date desc")
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties()
@@ -1,13 +1,13 @@
package com.fangjia.script.wenyan.seawater
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,13 +1,13 @@
package com.fangjia.script.wenyan.seawater
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,6 +1,8 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
@@ -9,294 +11,33 @@ import net.sf.json.JSONObject
import java.util.regex.Matcher
import java.util.regex.Pattern
//
//def map = [:]
//ExcelUtils.read("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批0829.xlsx", 1).each {List item->
// if(item["AF"] && item["AH"]){
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
// map.get(key, []) << key
// }
//}
//ExcelUtils.modify("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批(房价网交付0905).xlsx", { List item ->
//// if(item["AF"] && item["AH"]){
//// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
//// List nums = map.get(key)
//// item["AT"] = nums.size()
//// }
//
// def city = "上海"
// def addr = item["AC"]
// if(addr){
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
// println searchAddress
//// item["AD"] = searchAddress.province
//// item["AE"] = searchAddress.city
//// item["AF"] = searchAddress.district
// if(searchAddress.standardAddress){
//// item["AG"] = searchAddress.street
//// if(searchAddress.streetNumber){
//// item["AG"] = item["AG"] + searchAddress.streetNumber
//// }
//
// if(searchAddress.township){
// item["AP"] = searchAddress.township
// }
// }
//
// if(item["AG"]){
// def addrMap = getRoad(item["AG"])
// if(addrMap){
// if(addrMap.road){
// item["AQ"] = addrMap.road
// }
// if(addrMap.roadNumber){
// item["AR"] = addrMap.roadNumber
// }
// }
// }
// }
//
//
//// if (item["AH"] && item["AF"]) {
//// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["AE"], item["AF"], item["AH"]))
//// if (seaweed) {
//// if (item["AR"] && item["AS"]) {
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2, item["AS"].toInteger(), item["AR"].toInteger())
//// if (eva) {
//// item["AL"] = eva.avgPrice
//// item["AM"] = eva.totalPrice
//// }
//// } else {
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2)
//// if (eva) {
//// item["AL"] = eva.avgPrice
//// item["AM"] = eva.totalPrice
//// }
//// }
//// }
////
//// }
//
//},1)
// if(item["AK"] =~ /^\d{4}$/){
// item["AR"] = item["AK"].toString().substring(0, 2)
// }else if(item["AK"] =~ /^\d{3}$/){
// item["AR"] = item["AK"].toString().substring(0, 1)
// }
// if(item["AR"] && item["AH"]){
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AS"] = nums[nums.size() - 1]
// item["AT"] = nums.unique().size()
// }
// println(item)
// item["AG"] = null
// def city = "上海"
// def addr = item["AC"]
// def area = item["o"]
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
// item["AD"] = searchAddress.province
// item["AE"] = searchAddress.city
// item["AF"] = searchAddress.district
// if(searchAddress.standardAddress){
// item["AG"] = searchAddress.street
// if(searchAddress.streetNumber){
// item["AG"] = item["AG"] + searchAddress.streetNumber
// }
// }
//
//
// if(resp.getDouble("credit") > 0){
// item["AE"] = resp.city
// item["AF"] = resp.district
// item["AH"] = resp.name
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
// item["AL"] = eva.avgPrice
// item["AM"] = eva.totalPrice
// item["AN"] = new Date().format("yyyy-MM-dd")
// if(!item["AG"]){
// item["AG"] = resp.address
// }
// }
// item["AI"] = searchAddress.buildingNumber
// item["AJ"] = searchAddress.unitNumber
// item["AK"] = searchAddress.roomNumber
// println(item)
//},1)
//
def sheetIndexs = ["湘潭": 0, "嘉兴": 1, "苏州": 2]
sheetIndexs.each { String city, int sheetIndex ->
// def map = [:]
// ExcelUtils.read("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批0828(测试)1.xlsx", sheetIndex).each {List item->
// if(item["Q"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// map.get(key, []) << key
// }
// }
ExcelUtils.multiThreadingModify("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批(房价网交付0905).xlsx", { List item ->
// if(item["Q"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AE"] = nums.size()
// }
// item["W"] = null
// item["X"] = null
// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["P"], item["Q"], item["S"]))
// if(seaweed){
// if(item["AC"] && item["AD"]){
// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2, item["AD"].toInteger(), item["AC"].toInteger())
// if(eva){
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// }
// }else{
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2)
// if(eva){
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// }
// }
// }
// }
// if(item["AC"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AD"] = nums[nums.size() - 1]
// item["AE"] = nums.unique().size()
// }
// if(item["V"] =~ /^\d{4}$/){
// item["AC"] = item["V"].toString().substring(0, 2)
// }else if(item["V"] =~ /^\d{3}$/){
// item["AC"] = item["V"].toString().substring(0, 1)
// }else if(item["V"] =~ /^\d{6,7}$/){
// item["AC"] = item["V"].toString().substring(0, 2).replaceAll("^0", "")
// }
def addr = item["B"].toString().replaceAll("(", "").replace('区/县', "区").replace("街道/镇", "镇")
.replaceAll(")", "").replaceAll("【.*】", "").replaceAll("室/单元", "号")
// def area = item["L"]
def region = null
if (city == "嘉兴") {
region = "嘉善"
} else if (city == "苏州") {
region = "昆山"
}
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property", false, region)
JSONObject searchAddress = resp.getJSONObject("searchAddress")
def township = ""
if(searchAddress && searchAddress.township){
item["AP"] = searchAddress.township
township = searchAddress.township
}
if(item["S"]){
def addrMap = getRoad(item["S"])
if(addrMap){
if(addrMap.road){
def s = addrMap.road;
if(township){
if(township.contains(s)){
item["AQ"] = s.replace(township, "")
}else{
if(township.contains("镇")){
township = township.replace("镇","")
}
item["AQ"] = s.replace(township, "")
}
}else{
item["AQ"] = addrMap.road
}
}
if(addrMap.roadNumber){
item["AR"] = addrMap.roadNumber
}
}
}
println searchAddress
// item["O"] = searchAddress.province
// item["P"] = searchAddress.city
// item["Q"] = searchAddress.district
// if (searchAddress.standardAddress) {
// item["R"] = searchAddress.street
// if (searchAddress.streetNumber) {
// item["R"] = item["R"] + searchAddress.streetNumber
// }
// }
// if (resp.getDouble("credit") > 0) {
// item["O"] = resp.province
// item["P"] = resp.city
// item["Q"] = resp.district
// item["S"] = resp.name
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// item["Y"] = new Date().format("yyyy-MM-dd")
// if (!item["R"]) {
// item["R"] = resp.address
// }
// }
// item["T"] = searchAddress.buildingNumber
// item["U"] = searchAddress.unitNumber
// item["V"] = searchAddress.roomNumber
// if(city == "湘潭"){
// String roomNumber = DataUtil.match(addr, Pattern.compile("\\d+(号|(号房))\$"))
// if(roomNumber){
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
// }
// roomNumber = DataUtil.match(addr, Pattern.compile("\\d{6,8}\$"))
// if(roomNumber){
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
// }
//
// }
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
new File("D:\\Êý¾Ý\\ѧУÊý¾Ý").eachFile {File file->
ExcelUtils.read(file.getAbsolutePath(), 0).each {List item->
BasicDBObject dbo = new BasicDBObject()
dbo.city = item["A"]
dbo.region = item["B"]
dbo.name = item["C"]
dbo.address = item["E"]
dbo.attribute = item["G"]
dbo.property = item["H"]
dbo.category = "Сѧ"
dbo.location = [DataUtil.roundDouble(item["I"], 6), DataUtil.roundDouble(item["J"], 6)]
school.insert(dbo)
println(dbo)
}
ExcelUtils.read(file.getAbsolutePath(), 1).each {List item->
BasicDBObject dbo = new BasicDBObject()
dbo.city = item["A"]
dbo.region = item["B"]
dbo.name = item["C"]
dbo.address = item["E"]
dbo.attribute = item["G"]
dbo.property = item["H"]
dbo.category = "ÖÐѧ"
dbo.location = [item["I"], item["J"]]
school.save(dbo)
println(item)
}, sheetIndex)
}
def getRoad(def s){
def rex = ".*(路|街道|街|大道|巷|道|里).*\\d+(弄|号)\$"
Pattern pattern = Pattern.compile(rex);
Matcher matcher = pattern.matcher(s);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list && list.size()>0){
def s1 = list[0]
def road = getRexText(s1, ".*(路|街道|街|大道|巷|道|里)")
def num = getRexText(s1, "\\d+")
return ["road":road, "roadNumber":num]
}else{
def road = getRexText(s, ".*(路|街道|街|大道|巷|道|里)")
return ["road":road]
}
return null
}
def getRexText(s, regx){
Pattern pattern = Pattern.compile(regx);
Matcher matcher = pattern.matcher(s);
while (matcher.find()) {
return matcher.group();
}
return null;
}
@@ -1,9 +1,16 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang3.StringUtils
import org.openqa.selenium.json.Json
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
@@ -30,8 +37,6 @@ import org.apache.commons.lang3.StringUtils
//}, 0,"结果")
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
// if (item["T"] == null || ![1, 2, 4].contains(item["T"].toString().toDouble().toInteger())) {
// return
@@ -114,43 +119,80 @@ import org.apache.commons.lang3.StringUtils
// println("${key} ${names.join(",")}")
// }
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\µØÖ·±ê×¼»¯.xlsx",{List item->
String address = "";
if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
address = ProvinceData.getDisplay(item["B"])
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\地址标准化.xlsx",{List item->
// String address = "";
// if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
// address = ProvinceData.getDisplay(item["B"])
// }
// if(StringUtils.isNotBlank(item["C"])){
// String cityDisplay = CityData.getCityDisplay(item["C"]);
// if(cityDisplay != null){
// address = address + cityDisplay
// }else{
// address = address + item["C"] + "市"
// }
// }
// if(StringUtils.isNotBlank(item["D"])){
// String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
// if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
// address = address + orgDistrict
// }
// }
// if(StringUtils.isNotBlank(item["E"])){
// address = address + item["E"]
// }
// if(StringUtils.isNotBlank(item["F"])){
// address = address + item["F"]
// }
// if(StringUtils.isNotBlank(item["G"])){
// address = address + item["G"]
// }
// if(StringUtils.isNotBlank(item["H"])){
// address = address + item["H"] + "栋"
// }
// if(StringUtils.isNotBlank(item["i"])){
// address = address + item["i"] + "单元"
// }
// if(StringUtils.isNotBlank(item["J"])){
// address = address + item["J"] + "室"
// }
// item["K"] = address
// println(address)
//})
def school = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia", "guiding_price")
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\深圳市住建局二手房指导价.xlsx", { List item ->
if (item["G"]) {
BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["G"], item["H"], item["I"]))
BasicDBObject guid = new BasicDBObject()
guid.guiding_type = 0
guid.estate_type = seaweed.estate_type2
guid.house_type = 0
guid.c_date = new Date()
guid.guiding_person = "深圳房管局"
guid.guiding_date = Date.parse('yyyyMMdd', '20210411')
guid.category = "district"
guid.guiding_value = (int)item["E"]
guid.name = item["D"]
guid.city = item["G"]
guid.region = item["H"]
guid.name = item["i"]
school.insert(guid)
println(guid)
}
if(StringUtils.isNotBlank(item["C"])){
String cityDisplay = CityData.getCityDisplay(item["C"]);
if(cityDisplay != null){
address = address + cityDisplay
}else{
address = address + item["C"] + "ÊÐ"
}
}
if(StringUtils.isNotBlank(item["D"])){
String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
address = address + orgDistrict
}
}
if(StringUtils.isNotBlank(item["E"])){
address = address + item["E"]
}
if(StringUtils.isNotBlank(item["F"])){
address = address + item["F"]
}
if(StringUtils.isNotBlank(item["G"])){
address = address + item["G"]
}
if(StringUtils.isNotBlank(item["H"])){
address = address + item["H"] + "¶°"
}
if(StringUtils.isNotBlank(item["i"])){
address = address + item["i"] + "µ¥Ôª"
}
if(StringUtils.isNotBlank(item["J"])){
address = address + item["J"] + "ÊÒ"
}
item["K"] = address
println(address)
// if(!item["F"]){
// JSONObject resp = OpenApiClient.addressMatch("深圳", item["D"], 'property', false, item["B"])
// if(resp.id){
// BasicDBObject seaweed = DbUtils.seaweed.findById(resp.id)
// if(seaweed){
// item["F"] = seaweed._id.toString()
// item["G"] = seaweed.city
// item["H"] = seaweed.region
// item["I"] = seaweed.name
// }
// println(item)
// }
//
// }
})
@@ -1,82 +1,149 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.HouseValidator
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import net.sf.json.JSONObject
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
ExcelUtils.multiThreadingModify("D:\\商务数据\\建设银行\\副本5市成交案例(测试).xlsx",{List item->
// if(item["R"]){
// return
// }
// def calcDate = null
// if(item["E"]){
// calcDate = item["E"].toString().substring(0, 10)
// }else{
// calcDate = item["B"].toString().substring(0, 10)
// }
// def floor = item["I"].toString().toDouble().toInteger().toString()
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
// def params =[:]
// params.city = item["F"]
// params.district = item["p"]
// params.name = item['Q']
// params.size = item["C"].toString()
// params.floor = floor
// params.maxFloor = maxFloor
// params.direction = dir
// params.decoration = fit
// params.calDate = calcDate
// params.__refresh_cache = "1"
// params.scope = "1"
// JSONObject evaResp = OpenApiClient.evaluate(params)
// if(evaResp.result && evaResp.result.credit > 0.4){
// item["R"] = evaResp.result.totalPrice
// }
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
//school.find(new BasicDBObject("source", "查学区")).each {BasicDBObject item->
// item.category = item.attribute
// item.attribute = null
// school.save(item)
// println(item)
// if(item["H"] != "中粮云景花园"){
// return
//}
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\无经纬度学校-结果.xlsx").each {List item->
// if(item["P"] != null){
// def location = item["P"].split(",") as List
// def data = new BasicDBObject()
// data.map_name = item["N"]
// data.map_region = item["R"]
// data.map_address = item["N"]
// data.map_type = item["S"]
// data.location = [location[0].toString().toDouble(),location[1].toString().toDouble()]
// school.update(new BasicDBObject("_id", new ObjectId(item["A"])), new BasicDBObject('$set', data))
// println(data)
// }
def city = item["F"]
def region = item["G"]
def name = item["H"].toString()
def addr = item["D"]
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property" , false, region, name)
if(resp.name){
item["p"] = resp.district
item['Q'] = resp.name
//}
regionCodeMap = [:]
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
for(def city : cityCodeMap.keySet()){
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
for(def d : datas){
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
}
println(item)
// def calcDate = null
// if(item["E"]){
// calcDate = item["E"].toString().substring(0, 10)
// }else{
// calcDate = item["B"].toString().substring(0, 10)
// }
// def floor = item["I"].toString().toDouble().toInteger().toString()
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
// def params =[:]
// params.city = resp.city
// params.district = resp.district
// params.name = resp.name
// params.size = item["C"].toString()
// params.floor = floor
// params.maxFloor = maxFloor
// params.direction = dir
// params.decoration = fit
// params.calDate = calcDate
// params.scope = "1"
// JSONObject evaResp = OpenApiClient.evaluate(params)
// if(evaResp.result && evaResp.result.credit > 0.4){
// item["R"] = evaResp.result.totalPrice
// }
// println(item)
// }
}
def data = Collections.synchronizedList([])
data << ["ID","CITYID","DISTID","COMMNAME","LNG","LAT","ADDRESS","PS","PS_ADDR","PS_TYPE","PS_SEQNO","PS_LONGITUDE","PS_LATITUDE","MS","MS_ADDR","MS_TYPE","MS_SEQNO","MS_LONGITUDE","MS_LATITUDE","REMARK","CREATETIME","UPDATETIME","SOURCECODE"]
cityCodeMap.keySet().each {def city->
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
if(seaweed.lng){
def item = []
item << seaweed._id.toString()
item << cityCodeMap.get(seaweed.city)
item << regionCodeMap.get(seaweed.city + "-" + seaweed.region)
item << seaweed.name
def lng = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
def lat = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
item << lng
item << lat
item << seaweed.address
BasicDBObject location = new BasicDBObject()
BasicDBObject near = new BasicDBObject()
BasicDBObject geometry = new BasicDBObject("type", "Point")
geometry.append("coordinates", [lng, lat])
near.append('$maxDistance', 1000)
near.append('$geometry', geometry)
location.append('$near', near)
BasicDBObject query = new BasicDBObject()
query.append("location", location)
query.append("attribute",new BasicDBObject('$in', ['省重点', '市重点', '区重点', '县重点', '省级示范']))
query.append("category", new BasicDBObject('$in', ['中学']))
BasicDBObject midSchool = school.findOne(query)
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
// query.append("location", location)
// query.append("attribute", level)
// midSchool = school.findOne(query)
// if(midSchool){
// break
// }
// }
// if(!midSchool){
// query.removeField("attribute")
// midSchool = school.findOne(query)
// }
BasicDBObject primarySchool=null
query.append("category", new BasicDBObject('$in', ['小学']))
primarySchool = school.findOne(query)
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
// query.append("location", location)
// query.append("attribute", level)
// primarySchool = school.findOne(query)
// if(primarySchool){
// break
// }
// }
// if(!primarySchool){
// query.removeField("attribute")
// primarySchool = school.findOne(query)
// }
if(primarySchool){
item << primarySchool.name
item << primarySchool.address
item << primarySchool.attribute
item << null
item << primarySchool.location[0]
item << primarySchool.location[1]
}else {
item << null
item << null
item << null
item << null
item << null
item << null
}
if(midSchool){
item << midSchool.name
item << midSchool.address
item << midSchool.attribute
item << null
item << midSchool.location[0]
item << midSchool.location[1]
}else {
item << null
item << null
item << null
item << null
item << null
item << null
}
item << null
Date c_date = seaweed.c_date
Date m_date = seaweed.m_date
if(!c_date && !m_date){
c_date = DateUtil.add(new Date(), Calendar.YEAR, -1)
}
if(!c_date){
c_date = m_date
}
if(!m_date){
m_date = c_date
}
item <<c_date.format("yyyy-MM-dd HH:mm:ss")
item <<m_date.format("yyyy-MM-dd HH:mm:ss")
item << 'fjw'
println(item)
if(primarySchool || midSchool){
data << item
}
}
})
}
ExcelUtils.write("D:\\无学区房数据.xlsx", data, "结果")
})
@@ -1,20 +1,12 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq
//
//def data = [["城市", "区域", "小区名", "2018", "全部"]]
//cityList = ["上海"]
@@ -47,10 +39,6 @@ import static com.mongodb.client.model.Filters.eq
//HttpRequest httpRequest = new HttpRequest()
//println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8")
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
MongoCollection<Document> AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config")
AVERAGER_CONFIG.find().each {Document doc->
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20201101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20201201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -1,19 +1,13 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import com.mongodb.client.MongoCollection
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -61,7 +55,7 @@ import java.sql.Statement
//println(bigMarkupCount)
def countHouses(String city, String region, String name){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,28 +1,16 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.bson.Document
import com.mongodb.client.MongoCollection
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def poiData = Collections.synchronizedList([])
//
//ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item ->
@@ -79,7 +67,7 @@ def poiData = Collections.synchronizedList([])
// println(item)
//},1)
def findHouseCount(String city, String region, String name, List alias) {
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -11,26 +11,44 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
def data = []
def title = ["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg"]
def param_keys = ["tid", "city", "district", "address", "name", "size", "propertyType"]
param_keys.each {
title << "参数_${it}"
}
def resMap_keys = ["city", "district", "name", "avgPrice", "totalPrice", "buildYear"]
resMap_keys.each {
title << "结果_${it}"
}
data << title
def keys = new HashSet()
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if(!result.buildYear || !result.avgPrice){
if (!result.buildYear || !result.avgPrice) {
return
}
def params = JSONObject.fromObject(doc.parameter_map)
keys.addAll(params.keys())
def key = "${params.address}_${params.size}".toString()
if (!hashSet.contains(key)) {
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
param_keys.each {
item << params[it]
}
resMap_keys.each {
item << result[it]
}
data << item
hashSet << key
} else {
@@ -1,11 +1,17 @@
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
//import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
@@ -132,4 +138,70 @@ import org.apache.commons.lang3.StringUtils
// item["E"] = RegionData.formatRegion(item["B"], item["C"])
// println(item)
//})
CommonUtils.reFitAverage("常州","新北","龙城花园", "普通住宅")
//def border = [13511506.7,3639168.61,13511516.8,3639404.86,13511515.27,3639416.92,13511738.21,3639384.94,13512007.72,3639336.35,13511984.92,3639074.96,13511722.75,3639121.08,13511506.7,3639168.61]
//def coords = []
//for(int i=0; i < border.size() - 1; i=i+2){
// coords << "${border[i]},${border[i+1]}"
//}
//def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
//println(json)
//def points = []
//for(JSONObject it : json.result){
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// points << "${geo.lng},${geo.lat}"
// println(" new AMap.LngLat(${geo.lng},${geo.lat}),")
//}
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
//BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
// def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
// def coords = []
// def baidu_points = []
// def gaode_points = []
// for(int i=0; i < border.size() - 1; i=i+2){
// coords << "${border[i]},${border[i+1]}"
// if(coords.size() > 50){
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
// for(JSONObject it : json.result){
// baidu_points << "${it.x},${it.y}"
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
// }
// coords.clear()
// }
// }
// if(coords.size() > 0){
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
// for(JSONObject it : json.result){
// baidu_points << "${it.x},${it.y}"
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
// }
// }
// dbo.format_city = RegionData.formatCity(dbo.city)
// dbo.baidu_border = baidu_points.join(";")
// dbo.gaode_border = gaode_points.join(";")
// BAIDUMAP_BORDER_COLL.save(dbo)
// println(dbo)
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Ð¡ÇøÈºÏñ²âÊÔÊý¾Ý.xlsx",{List item->
def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
if(seaweed){
// item["A"] = seaweed._id.toString()
def names = [item['D']]
if(seaweed.alias){
names.addAll(seaweed.alias)
}
names.unique()
// if(seaweed.lng){
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
// }
def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
if(one){
item["I"] = one.gaode_border
}
println(item)
}
})
@@ -669,7 +669,6 @@ def findRentInfoByRoom(String city, String region){
def findRent(def city, def region, def name){
DBCollection DISTRICT_STAT_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
def query = new BasicDBObject()
query.city = city
query.region = region

Some files were not shown because too many files have changed in this diff Show More